Compare commits
37
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9deaee9eea | ||
|
|
22d4cbd144 | ||
|
|
7b618b2317 | ||
|
|
d8477c71b1 | ||
|
|
45fae9da8a | ||
|
|
2f742fd3fe | ||
|
|
82180bffd2 | ||
|
|
ac82187b85 | ||
|
|
8d38c6bac9 | ||
|
|
9afef800c0 | ||
|
|
eaf27f36af | ||
|
|
4efe6cfb26 | ||
|
|
3e7f852472 | ||
|
|
5ba6881f9f | ||
|
|
61196c99e6 | ||
|
|
250364a57b | ||
|
|
73779a5dd1 | ||
|
|
ec76ab77c0 | ||
|
|
91d041b62a | ||
|
|
51444a8341 | ||
|
|
dd9138310c | ||
|
|
653da54c0d | ||
|
|
f72f57f6e2 | ||
|
|
975f0ed654 | ||
|
|
706169aa8a | ||
|
|
6d2d50b1e1 | ||
|
|
3d7d65edce | ||
|
|
06fd53e5cb | ||
|
|
81182e9dcd | ||
|
|
f7b23033a2 | ||
|
|
cf26e95691 | ||
|
|
8daa5977f9 | ||
|
|
30ca8a22bc | ||
|
|
97e605edb0 | ||
|
|
7ef796b04b | ||
|
|
ec93585736 | ||
|
|
6b63f13da6 |
@@ -19,6 +19,7 @@ import (
|
|||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"git.novox.be/novox/mesh-sdk/go/process"
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
"git.novox.be/novox/mesh-sdk/go/provider"
|
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||||
@@ -125,8 +126,8 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
|
|||||||
return done, stopped
|
return done, stopped
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable,
|
// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
|
||||||
// having opened every bucket it names.
|
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
|
||||||
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||||
b := startBed(t, "")
|
b := startBed(t, "")
|
||||||
done := b.start(t)
|
done := b.start(t)
|
||||||
@@ -150,9 +151,21 @@ func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
|||||||
}
|
}
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != 4 || !said.Readable("asks") ||
|
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||||
!said.Readable("wants") || !said.Readable("differences") || !said.Readable("compositions") {
|
if err != nil {
|
||||||
t.Fatalf("the assigner said %+v, want its four states, each readable", said)
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m, err := catalogue.ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != len(m.State) {
|
||||||
|
t.Fatalf("the assigner said %+v, want the %d states its manifest declares", said, len(m.State))
|
||||||
|
}
|
||||||
|
for _, s := range m.State {
|
||||||
|
if !said.Readable(s.Name) {
|
||||||
|
t.Errorf("the assigner said %+v: its state %s is not readable", said, s.Name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -281,9 +294,12 @@ func (b *bed) get(t *testing.T, bucket, key string) []byte {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1297/B4 on the bus, 1906/D5: the controller sends ace and g14 builds while the canary ace does not run F healthy yet.
|
// 1297/B4 on the bus, 1906/D5: the controller sends ace, g14 and shanks builds while the canary ace does not run F healthy yet.
|
||||||
// Started whole, the assigner wants F on the canary, holds g14 back with no want, and writes no difference at all: there
|
// Started whole, the assigner wants F on the canary and writes no difference at all: there is no send to compare with (ADR 0539
|
||||||
// is no send to compare with (ADR 0539 §8).
|
// §8). 1945/AC19, AC21: shanks, which runs the older fingerprint (composed and granted) with no want, gets its first want at that
|
||||||
|
// fingerprint, never at F; g14, which runs a build of the old form, gets none, so web is on neither list there and the assigner
|
||||||
|
// raises the condition naming g14, web and the canary rule; once ace runs F healthy, g14's first want names F and the condition is
|
||||||
|
// cleared.
|
||||||
func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||||
b := startBed(t, "")
|
b := startBed(t, "")
|
||||||
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||||
@@ -296,19 +312,21 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||||
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
for _, fp := range []string{fNew, fOld} {
|
||||||
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
gkey, _ := delivery.ControllerGrantKey("web", fp)
|
||||||
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||||
b.compose(t, "ace", "web", fNew, 'c', delivery.JudgedNeither)
|
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||||
b.compose(t, "g14", "web", fNew, 'c', delivery.JudgedNeither)
|
}
|
||||||
for _, node := range []string{"ace", "g14"} {
|
runs := map[string]string{"ace": fOld, "g14": "build-9", "shanks": fOld}
|
||||||
|
for _, node := range []string{"ace", "g14", "shanks"} {
|
||||||
|
b.compose(t, node, "web", fNew, 'c', delivery.JudgedNeither)
|
||||||
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||||
b.putJSON(t, delivery.PresenceBucket, node, on)
|
b.putJSON(t, delivery.PresenceBucket, node, on)
|
||||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
have, _ := delivery.EncodeHave(delivery.Have{FP: runs[node], Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||||
b.putJSON(t, delivery.HaveBucket, node+".web", have)
|
b.putJSON(t, delivery.HaveBucket, node+".web", have)
|
||||||
|
b.putJSON(t, provider.AssignmentsBucket, node+".web", []byte(`{"node":"`+node+`","module":"web","build":"b-`+node+`"}`))
|
||||||
}
|
}
|
||||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`))
|
b.compose(t, "shanks", "web", fOld, 'a', delivery.JudgedNeither)
|
||||||
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`))
|
|
||||||
b.start(t)
|
b.start(t)
|
||||||
|
|
||||||
wkey, _ := delivery.WantKey("ace", "web")
|
wkey, _ := delivery.WantKey("ace", "web")
|
||||||
@@ -316,12 +334,41 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
|||||||
if err != nil || want.FP != fNew {
|
if err != nil || want.FP != fNew {
|
||||||
t.Fatalf("the canary ace's want = %+v, %v", want, err)
|
t.Fatalf("the canary ace's want = %+v, %v", want, err)
|
||||||
}
|
}
|
||||||
time.Sleep(500 * time.Millisecond)
|
if w := b.wantNow(t, "shanks", "web", "shanks' first want at what it runs", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fOld || w.Composed != "sha256:"+strings.Repeat("a", 64) || w.Newest != fNew {
|
||||||
|
t.Errorf("shanks' want = %+v, want the fingerprint it runs naming its composition, held back from F", w)
|
||||||
|
}
|
||||||
|
var neither asks.Condition
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && neither.Key == ""; time.Sleep(100 * time.Millisecond) {
|
||||||
|
for k, c := range b.conditionsSaid(t) {
|
||||||
|
if strings.Contains(k, ".neither.") && c.State == asks.StateRaised {
|
||||||
|
neither = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if neither.Subject.Machine != "g14" || neither.Subject.ID != "web" || neither.Severity != asks.SeverityUrgent || !strings.Contains(neither.Explanation, "canary") {
|
||||||
|
t.Errorf("the condition of web on neither list on g14 = %+v", neither)
|
||||||
|
}
|
||||||
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
|
if keys, err := b.buckets[delivery.AssignerDifferencesBucket].Keys(b.ctx); err == nil && len(keys) != 0 {
|
||||||
t.Errorf("differences were written: %v", keys)
|
t.Errorf("differences were written: %v", keys)
|
||||||
}
|
}
|
||||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
|
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
|
||||||
t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value())
|
t.Errorf("g14, which runs an old-form build and is held back by the canary, was given a want: %s", e.Value())
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC21: once the canary ace runs F healthy, g14's first want names F by the rules as usual, and web leaves neither list
|
||||||
|
// there by itself: the condition is cleared.
|
||||||
|
healthyF, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||||
|
b.putJSON(t, delivery.HaveBucket, "ace.web", healthyF)
|
||||||
|
if w := b.wantNow(t, "g14", "web", "g14's first want at F once the canary runs it healthy", func(w delivery.Want) bool { return w.FP != "" }); w.FP != fNew || w.Composed != "sha256:"+strings.Repeat("c", 64) || w.Newest != "" {
|
||||||
|
t.Errorf("g14's want = %+v, want F with its composition", w)
|
||||||
|
}
|
||||||
|
cleared := false
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !cleared; time.Sleep(100 * time.Millisecond) {
|
||||||
|
c, ok := b.conditionsSaid(t)[neither.Key]
|
||||||
|
cleared = ok && c.State == asks.StateCleared
|
||||||
|
}
|
||||||
|
if !cleared {
|
||||||
|
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -341,12 +388,14 @@ func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
|||||||
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||||
b.start(t)
|
b.start(t)
|
||||||
hkey, _ := delivery.HeldKey("web", fNew)
|
hkey, _ := delivery.HeldKey("web", fNew)
|
||||||
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" {
|
// 1906/H3: the held record names the machine and keeps the health, when and the node's words.
|
||||||
|
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" ||
|
||||||
|
!strings.HasPrefix(h.Words, "health failed, phase running, seen at ") || !strings.HasSuffix(h.Words, ": connection refused") {
|
||||||
t.Fatalf("held = %+v, %v", h, err)
|
t.Fatalf("held = %+v, %v", h, err)
|
||||||
}
|
}
|
||||||
akey, _ := delivery.HoldAskKey("web", fNew)
|
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||||
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||||
if err != nil || a.Ask.ID != delivery.AskingID(akey, 0) || len(a.Ask.Options) != 2 {
|
if err != nil || a.Asking <= 0 || a.Ask.ID != delivery.AskingID(akey, a.Asking) || len(a.Ask.Options) != 2 {
|
||||||
t.Fatalf("the ask = %+v, %v", a, err)
|
t.Fatalf("the ask = %+v, %v", a, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -413,7 +462,7 @@ func TestAFailedDataModuleIsAskedToRestore(t *testing.T) {
|
|||||||
key := ""
|
key := ""
|
||||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && key == ""; time.Sleep(100 * time.Millisecond) {
|
||||||
for k, c := range b.conditionsSaid(t) {
|
for k, c := range b.conditionsSaid(t) {
|
||||||
if c.State == asks.StateRaised && c.Subject.ID == "mail" {
|
if c.State == asks.StateRaised && c.Subject.ID == "mail" && strings.Contains(k, ".held.") {
|
||||||
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
|
if c.Subject.Machine != "ace" || !strings.Contains(c.Explanation, "connection refused") {
|
||||||
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
|
t.Fatalf("the condition = %+v, want it to name ace and the check's words", c)
|
||||||
}
|
}
|
||||||
@@ -580,8 +629,15 @@ func (b *bed) wantNow(t *testing.T, node, module, what string, ok func(delivery.
|
|||||||
return last
|
return last
|
||||||
}
|
}
|
||||||
|
|
||||||
// answer says the teller wrote the operator's answer to the ask at key.
|
// answer says the teller wrote the operator's answer to the ask that stands at key (its warrant names that asking's id).
|
||||||
func (b *bed) answer(t *testing.T, key, option string) { b.answerTo(t, key, option, "r1") }
|
func (b *bed) answer(t *testing.T, key, option string) {
|
||||||
|
t.Helper()
|
||||||
|
a, err := delivery.DecodeAssignerAsk(key, b.get(t, delivery.AssignerAsksBucket, key))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.answerTo(t, key, option, a.Ask.ID)
|
||||||
|
}
|
||||||
|
|
||||||
// answerTo is answer to the ask with this id (the router's warrant names it).
|
// answerTo is answer to the ask with this id (the router's warrant names it).
|
||||||
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
|
func (b *bed) answerTo(t *testing.T, key, option, askID string) {
|
||||||
@@ -625,8 +681,8 @@ func TestAReleaseAnswerWritesTheWantAgain(t *testing.T) {
|
|||||||
}
|
}
|
||||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||||
cleared := false
|
cleared := false
|
||||||
for _, c := range b.conditionsSaid(t) {
|
for k, c := range b.conditionsSaid(t) {
|
||||||
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared)
|
cleared = cleared || (c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held."))
|
||||||
}
|
}
|
||||||
if cleared {
|
if cleared {
|
||||||
return
|
return
|
||||||
@@ -773,3 +829,81 @@ func TestAUserListRevisionOfTheBusIsWrittenAtOnce(t *testing.T) {
|
|||||||
t.Errorf("the user-list revision was asked")
|
t.Errorf("the user-list revision was asked")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1906/H2, whole on the bus: given a fingerprint held after one failed check, when every node that runs the module then runs
|
||||||
|
// it healthy, the assigner lifts the hold by itself: the held record and its ask are taken away, the held condition is
|
||||||
|
// cleared saying why, and the want stays at the fingerprint.
|
||||||
|
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||||
|
b := startBed(t, "")
|
||||||
|
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||||
|
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||||
|
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||||
|
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||||
|
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||||
|
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||||
|
b.start(t)
|
||||||
|
hkey, _ := delivery.HeldKey("web", fNew)
|
||||||
|
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||||
|
b.get(t, delivery.AssignerWantsBucket, hkey)
|
||||||
|
b.get(t, delivery.AssignerAsksBucket, akey)
|
||||||
|
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||||
|
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
||||||
|
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||||
|
if b.hasKey(delivery.AssignerWantsBucket, hkey) || b.hasKey(delivery.AssignerAsksBucket, akey) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for k, c := range b.conditionsSaid(t) {
|
||||||
|
if c.Subject.ID == "web" && c.State == asks.StateCleared && strings.Contains(k, ".held.") && strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||||
|
if w := b.wantNow(t, "ace", "web", "the want at the lifted fingerprint", func(w delivery.Want) bool { return w.FP == fNew }); w.Rule != 0 {
|
||||||
|
t.Errorf("the want = %+v", w)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("the hold was not lifted: held %v, ask %v", b.hasKey(delivery.AssignerWantsBucket, hkey), b.hasKey(delivery.AssignerAsksBucket, akey))
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/H2, whole on the bus: a hold lifted by itself and written again after the assigner was started again is asked under a
|
||||||
|
// new id, so the teller raises it; the answer to the first hold does not carry over (1906/H4).
|
||||||
|
func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
|
||||||
|
b := startBed(t, "")
|
||||||
|
fNew := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||||
|
b.fixture(t, "web", fNew, `{"module":"web","version":"1"}`)
|
||||||
|
b.compose(t, "ace", "web", fNew, 'b', delivery.JudgedNeither)
|
||||||
|
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||||
|
healthy, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||||
|
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||||
|
b.putWant(t, "ace", "web", delivery.Want{FP: fNew, Composed: "sha256:" + strings.Repeat("b", 64)})
|
||||||
|
_, stop := b.startStoppable(t)
|
||||||
|
hkey, _ := delivery.HeldKey("web", fNew)
|
||||||
|
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||||
|
first, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
||||||
|
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("the hold was not lifted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
stop()
|
||||||
|
time.Sleep(1100 * time.Millisecond) // the next hold is written in another second
|
||||||
|
b.answerTo(t, akey, "keep", first.Ask.ID) // an answer to the first hold, which the later hold must not take
|
||||||
|
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||||
|
b.start(t)
|
||||||
|
again, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if again.Ask.ID == first.Ask.ID {
|
||||||
|
t.Fatalf("the hold written again is asked under the first id %s", first.Ask.ID)
|
||||||
|
}
|
||||||
|
b.putJSON(t, delivery.HaveBucket, "ace.web", healthy)
|
||||||
|
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.AssignerWantsBucket, hkey); time.Sleep(100 * time.Millisecond) {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("the stale \"keep\" of the first hold kept the second one held")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -29,6 +29,15 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
if os.Getenv("MESH_BROKER_FILE") == "" {
|
||||||
|
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
|
||||||
|
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
|
||||||
|
if err := serveTools(); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, "mesh-delivery-assigner:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
defer stop()
|
defer stop()
|
||||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||||
@@ -59,15 +68,17 @@ var readBuckets = []string{
|
|||||||
const goBuildModule = "mesh-build-go"
|
const goBuildModule = "mesh-build-go"
|
||||||
|
|
||||||
// ownBuckets are the assigner's own states, as its manifest declares them.
|
// ownBuckets are the assigner's own states, as its manifest declares them.
|
||||||
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket}
|
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
|
||||||
|
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
|
||||||
|
|
||||||
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||||
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||||
// and the teller.
|
// and the teller, its puts by whoever follows one (ADR 0546).
|
||||||
var keptStates = []announce.State{
|
var keptStates = []announce.State{
|
||||||
{Name: "asks", Readable: true},
|
{Name: "asks", Readable: true},
|
||||||
{Name: "compositions", Readable: true},
|
{Name: "compositions", Readable: true},
|
||||||
{Name: "differences", Readable: true},
|
{Name: "differences", Readable: true},
|
||||||
|
{Name: "puts", Readable: true},
|
||||||
{Name: "wants", Readable: true},
|
{Name: "wants", Readable: true},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,11 +86,17 @@ var keptStates = []announce.State{
|
|||||||
type buckets map[string]jetstream.KeyValue
|
type buckets map[string]jetstream.KeyValue
|
||||||
|
|
||||||
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
|
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
|
||||||
// without it would write wants from half the mesh.
|
// without it would write wants from half the mesh. The puts' bucket is the exception: whatever keeps it from opening (it
|
||||||
func open(ctx context.Context, js jetstream.JetStream) (buckets, error) {
|
// does not exist yet, the credential is not granted it yet, the bus does not answer in time), the assigner writes wants
|
||||||
|
// without it and takes no put further until it starts again (ADR 0546).
|
||||||
|
func open(ctx context.Context, js jetstream.JetStream, logf func(string, ...any)) (buckets, error) {
|
||||||
out := buckets{}
|
out := buckets{}
|
||||||
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||||
kv, err := js.KeyValue(ctx, name)
|
kv, err := js.KeyValue(ctx, name)
|
||||||
|
if name == delivery.AssignerPutsBucket && err != nil {
|
||||||
|
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
|
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
|
||||||
}
|
}
|
||||||
@@ -118,7 +135,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
kvs, err := open(ctx, js)
|
kvs, err := open(ctx, js, logger.Printf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -133,9 +150,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||||
}
|
}
|
||||||
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||||
return follow(ctx, js, kvs, sayVia(ctx, js), logger)
|
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
|
||||||
|
type emitting struct{ *process.Conn }
|
||||||
|
|
||||||
|
func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(eventType, body) }
|
||||||
|
|
||||||
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||||
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||||
var watched = []string{
|
var watched = []string{
|
||||||
@@ -156,7 +178,7 @@ var watched = []string{
|
|||||||
|
|
||||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||||
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||||
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, logger *log.Logger) error {
|
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
|
||||||
w := newWorld()
|
w := newWorld()
|
||||||
dirty := make(chan struct{}, 1)
|
dirty := make(chan struct{}, 1)
|
||||||
wake := func() {
|
wake := func() {
|
||||||
@@ -165,13 +187,15 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
errs := make(chan error, len(watched)+1)
|
errs := make(chan error, len(watched)+2)
|
||||||
settled := make(chan struct{}, len(watched)+1)
|
settled := make(chan struct{}, len(watched)+2)
|
||||||
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
||||||
kw, err := kvs[name].WatchAll(ctx, opts...)
|
kw, err := kvs[name].WatchAll(ctx, opts...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
|
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
|
||||||
}
|
}
|
||||||
|
// The puts' watch never stops the wants (ADR 0546): when it ends, the puts are no longer heard, said in the journal.
|
||||||
|
optional := name == delivery.AssignerPutsBucket
|
||||||
go func() {
|
go func() {
|
||||||
defer kw.Stop()
|
defer kw.Stop()
|
||||||
for {
|
for {
|
||||||
@@ -181,6 +205,10 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
|||||||
return
|
return
|
||||||
case e, ok := <-kw.Updates():
|
case e, ok := <-kw.Updates():
|
||||||
if !ok {
|
if !ok {
|
||||||
|
if optional {
|
||||||
|
logger.Printf("the watch of %s ended: no put is taken further until the assigner starts again", name)
|
||||||
|
return
|
||||||
|
}
|
||||||
errs <- fmt.Errorf("the watch of %s ended", name)
|
errs <- fmt.Errorf("the watch of %s ended", name)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -199,6 +227,15 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
|||||||
}()
|
}()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
followed := watched
|
||||||
|
if kvs[delivery.AssignerPutsBucket] != nil {
|
||||||
|
if err := watch(delivery.AssignerPutsBucket); err != nil {
|
||||||
|
logger.Printf("%v: no put is taken further until the assigner starts again", err)
|
||||||
|
delete(kvs, delivery.AssignerPutsBucket)
|
||||||
|
} else {
|
||||||
|
followed = append(append([]string{}, watched...), delivery.AssignerPutsBucket)
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, name := range watched {
|
for _, name := range watched {
|
||||||
var opts []jetstream.WatchOpt
|
var opts []jetstream.WatchOpt
|
||||||
if name == delivery.HaveBucket {
|
if name == delivery.HaveBucket {
|
||||||
@@ -211,7 +248,7 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
|||||||
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for range len(watched) + 1 {
|
for range len(followed) + 1 {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return nil
|
return nil
|
||||||
@@ -222,6 +259,9 @@ func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(a
|
|||||||
}
|
}
|
||||||
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
|
||||||
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
|
||||||
|
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
|
||||||
|
p.acts = newPutActs(bus, wake, logger.Printf)
|
||||||
|
}
|
||||||
wake()
|
wake()
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -10,7 +12,10 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||||
@@ -48,6 +53,27 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
|
|||||||
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
|
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
|
||||||
t.Errorf("the program announces %v, keeps %v", got, want)
|
t.Errorf("the program announces %v, keeps %v", got, want)
|
||||||
}
|
}
|
||||||
|
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
|
||||||
|
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
|
||||||
|
for _, c := range []struct {
|
||||||
|
missing string
|
||||||
|
err error
|
||||||
|
starts bool
|
||||||
|
}{
|
||||||
|
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
|
||||||
|
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
|
||||||
|
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
|
||||||
|
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
|
||||||
|
} {
|
||||||
|
var logged string
|
||||||
|
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
|
||||||
|
switch {
|
||||||
|
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
|
||||||
|
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
|
||||||
|
case !c.starts && (err == nil || !strings.Contains(err.Error(), c.missing)):
|
||||||
|
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1297/B2a: without a credential the program does not start, and says why.
|
// 1297/B2a: without a credential the program does not start, and says why.
|
||||||
@@ -57,3 +83,17 @@ func TestNoCredential(t *testing.T) {
|
|||||||
t.Errorf("run without a credential = %v", err)
|
t.Errorf("run without a credential = %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refusingJS opens every bucket but one, which it refuses with err.
|
||||||
|
type refusingJS struct {
|
||||||
|
jetstream.JetStream
|
||||||
|
missing string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||||
|
if name == r.missing {
|
||||||
|
return nil, r.err
|
||||||
|
}
|
||||||
|
return &memKV{values: map[string][]byte{}}, nil
|
||||||
|
}
|
||||||
|
|||||||
BIN
Binary file not shown.
@@ -28,11 +28,26 @@ type passer struct {
|
|||||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||||
answeredAt map[string]uint64
|
answeredAt map[string]uint64
|
||||||
asks *composeAsks
|
asks *composeAsks
|
||||||
|
// now is the clock; nil is time.Now.
|
||||||
|
now func() time.Time
|
||||||
|
// neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the
|
||||||
|
// first pass after a start, which clears what a previous process may have raised (startClears).
|
||||||
|
neither map[string]neitherOne
|
||||||
|
// acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket).
|
||||||
|
acts *putActs
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *passer) clock() time.Time {
|
||||||
|
if p.now != nil {
|
||||||
|
return p.now()
|
||||||
|
}
|
||||||
|
return time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *passer) pass(ctx context.Context) {
|
func (p *passer) pass(ctx context.Context) {
|
||||||
p.compositions(ctx)
|
p.compositions(ctx)
|
||||||
p.wants(ctx)
|
p.wants(ctx)
|
||||||
|
p.puts(ctx)
|
||||||
p.nodeWants(ctx)
|
p.nodeWants(ctx)
|
||||||
p.backups(ctx)
|
p.backups(ctx)
|
||||||
}
|
}
|
||||||
@@ -78,6 +93,7 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
wanted := p.w.wantsOf()
|
wanted := p.w.wantsOf()
|
||||||
|
puts := p.putsByModule()
|
||||||
names := map[string]bool{}
|
names := map[string]bool{}
|
||||||
for name := range trunk {
|
for name := range trunk {
|
||||||
names[name] = true
|
names[name] = true
|
||||||
@@ -85,11 +101,23 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
for name := range wanted {
|
for name := range wanted {
|
||||||
names[name] = true
|
names[name] = true
|
||||||
}
|
}
|
||||||
|
neither := map[string]neitherOne{}
|
||||||
|
// The first pass after a start remembers nothing a previous process said: a neither or held condition it raised would stay
|
||||||
|
// raised for ever once the module has its want or is no longer held. That pass clears, once, the neither condition of every
|
||||||
|
// assigned module not on neither list and the held condition of every module not held; a clear of a condition that is not open is
|
||||||
|
// quiet at the router.
|
||||||
|
first := p.neither == nil
|
||||||
|
stale := map[string]neitherOne{}
|
||||||
|
defer func() { p.sayNeither(neither, stale) }()
|
||||||
for name := range names {
|
for name := range names {
|
||||||
fp := trunk[name].FP2
|
fp := trunk[name].FP2
|
||||||
name := name
|
name := name
|
||||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name],
|
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
|
||||||
|
GrantedAt: func(fp string) bool {
|
||||||
|
key, err := delivery.ControllerGrantKey(name, fp)
|
||||||
|
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
|
||||||
|
},
|
||||||
Restored: func(node string) bool {
|
Restored: func(node string) bool {
|
||||||
key, err := delivery.RestoreAskKey(node, name, fp)
|
key, err := delivery.RestoreAskKey(node, name, fp)
|
||||||
return err == nil && p.answeredWith(key, assign.OptionRestore)
|
return err == nil && p.answeredWith(key, assign.OptionRestore)
|
||||||
@@ -103,7 +131,13 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
grant, _ := delivery.ControllerGrantKey(name, fp)
|
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||||
hold, _ := delivery.HoldAskKey(name, fp)
|
hold, _ := delivery.HoldAskKey(name, fp)
|
||||||
m.Released = p.answeredWith(hold, "release")
|
m.Released = p.answeredWith(hold, "release")
|
||||||
|
m.Kept = p.answeredWith(hold, "keep")
|
||||||
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||||
|
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
|
||||||
|
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||||
|
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
|
||||||
|
}
|
||||||
|
}
|
||||||
if m.Released {
|
if m.Released {
|
||||||
p.clearHeld(name, fp)
|
p.clearHeld(name, fp)
|
||||||
}
|
}
|
||||||
@@ -113,6 +147,13 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
last[node] = p.w.lastHealthy(node, name, fp)
|
last[node] = p.w.lastHealthy(node, name, fp)
|
||||||
}
|
}
|
||||||
out := assign.Wants(m, nodes, wanted[name], last)
|
out := assign.Wants(m, nodes, wanted[name], last)
|
||||||
|
gatherNeither(neither, name, out)
|
||||||
|
if first {
|
||||||
|
startClears(stale, m, out)
|
||||||
|
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
|
||||||
|
p.clearHeld(name, fp)
|
||||||
|
}
|
||||||
|
}
|
||||||
if out.Held != nil {
|
if out.Held != nil {
|
||||||
key, err := delivery.HeldKey(name, fp)
|
key, err := delivery.HeldKey(name, fp)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@@ -120,8 +161,11 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
|
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if out.Held != nil || m.Held {
|
if out.Lifted {
|
||||||
p.holdAsk(ctx, name, fp)
|
p.lift(ctx, name, fp)
|
||||||
|
}
|
||||||
|
if out.Held != nil || (m.Held && !out.Lifted) {
|
||||||
|
p.holdAsk(ctx, name, fp, out.Held)
|
||||||
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
|
p.heldCondition(ctx, name, fp, out.Held, len(out.Restore) > 0)
|
||||||
}
|
}
|
||||||
for _, node := range out.Restore {
|
for _, node := range out.Restore {
|
||||||
@@ -142,6 +186,74 @@ func (p *passer) wants(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
|
||||||
|
type neitherOne struct {
|
||||||
|
node, module string
|
||||||
|
holding assign.Holding
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatherNeither adds the nodes on which module is on neither list in its outcome, keyed by node and module.
|
||||||
|
func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome) {
|
||||||
|
for node, h := range out.Neither {
|
||||||
|
into[node+"."+module] = neitherOne{node: node, module: module, holding: h}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
|
||||||
|
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
|
||||||
|
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
|
||||||
|
// for ever) or a move waiting for its record ask. Clearing a condition that is not open is quiet at the router.
|
||||||
|
func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome) {
|
||||||
|
for _, node := range m.Nodes {
|
||||||
|
if _, on := out.Neither[node]; !on {
|
||||||
|
into[node+"."+m.Name] = neitherOne{node: node, module: m.Name}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayNeither raises, once, the condition of every module on neither list in this pass and writes it to the journal, and clears,
|
||||||
|
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
|
||||||
|
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
|
||||||
|
// each is cleared once.
|
||||||
|
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
|
||||||
|
if p.neither == nil {
|
||||||
|
p.neither = map[string]neitherOne{}
|
||||||
|
for key, one := range stale {
|
||||||
|
if _, on := now[key]; !on {
|
||||||
|
p.neither[key] = one
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key, one := range now {
|
||||||
|
if _, said := p.neither[key]; said {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c := assign.NeitherCondition(one.node, one.module, one.holding, true)
|
||||||
|
if p.say != nil {
|
||||||
|
if err := p.say(c); err != nil {
|
||||||
|
p.logf("the condition of %s on neither list on %s could not be said, and is said at the next pass: %v", one.module, one.node, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
|
||||||
|
one.module, one.node, one.holding.Rule, one.holding.Words)
|
||||||
|
p.neither[key] = one
|
||||||
|
}
|
||||||
|
for key, one := range p.neither {
|
||||||
|
if _, still := now[key]; still {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if p.say != nil {
|
||||||
|
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
|
||||||
|
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.logf("%s on %s has a want again", one.module, one.node)
|
||||||
|
delete(p.neither, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
||||||
// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part.
|
// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part.
|
||||||
func (p *passer) compositions(ctx context.Context) {
|
func (p *passer) compositions(ctx context.Context) {
|
||||||
@@ -210,15 +322,62 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
|
|||||||
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
|
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
|
||||||
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
|
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
|
||||||
// node (wants are only compared, ADR 0429 §7), and the options say so.
|
// node (wants are only compared, ADR 0429 §7), and the options say so.
|
||||||
func (p *passer) holdAsk(ctx context.Context, module, fp string) {
|
func (p *passer) holdAsk(ctx context.Context, module, fp string, fresh *delivery.Held) {
|
||||||
key, err := delivery.HoldAskKey(module, fp)
|
key, err := delivery.HoldAskKey(module, fp)
|
||||||
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
a := assign.HoldAsk(key, module, fp, time.Now())
|
// The asking is the hold's time in seconds: a hold of the same fingerprint written again after a lift is another asking, under
|
||||||
|
// an id of its own, also after a restart, since the router drops a second ask under an id it keeps (issue 1906).
|
||||||
|
now := p.clock()
|
||||||
|
asking := int(now.Unix())
|
||||||
|
h := p.heldHold(module, fp, fresh)
|
||||||
|
a := assign.HoldAsk(key, module, fp, h.Node, h.Words, asking, now)
|
||||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// lift takes away a hold the assigner lifted by itself, since every node that runs the module runs the held fingerprint healthy
|
||||||
|
// (issue 1906): the held record and its ask (the teller then takes the ask back from the operator), and the held condition is
|
||||||
|
// cleared saying why, once. A hold of the same fingerprint written later is said and asked again.
|
||||||
|
func (p *passer) lift(ctx context.Context, module, fp string) {
|
||||||
|
held, err := delivery.HeldKey(module, fp)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ask, err := delivery.HoldAskKey(module, fp)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c := assign.LiftedCondition(module, fp)
|
||||||
|
if !p.said[c.Key+".lifted"] {
|
||||||
|
if p.say != nil {
|
||||||
|
if err := p.say(c); err != nil {
|
||||||
|
// The hold stays written, so the next pass lifts it again and says so then.
|
||||||
|
p.logf("the condition of the lifted hold of %s could not be cleared, and is cleared at the next pass: %v", module, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.logf("%s", assign.LiftedWords(module, fp))
|
||||||
|
p.said[c.Key+".lifted"] = true
|
||||||
|
delete(p.said, c.Key+".raised") // a hold written again is said again
|
||||||
|
}
|
||||||
|
p.remove(ctx, delivery.AssignerWantsBucket, held)
|
||||||
|
p.remove(ctx, delivery.AssignerAsksBucket, ask)
|
||||||
|
}
|
||||||
|
|
||||||
|
// remove deletes one key of the assigner's own bucket when it holds it; a delete that fails is said and made again at the next
|
||||||
|
// pass.
|
||||||
|
func (p *passer) remove(ctx context.Context, bucket, key string) {
|
||||||
|
if !p.w.has(bucket, key) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := p.kvs[bucket].Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
|
||||||
|
p.logf("%s %s could not be taken away: %v", bucket, key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.w.put(bucket, key, nil)
|
||||||
|
}
|
||||||
|
|
||||||
// heldHold is the hold of fp as written: the node whose health check failed and its words.
|
// heldHold is the hold of fp as written: the node whose health check failed and its words.
|
||||||
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
|
func (p *passer) heldHold(module, fp string, fresh *delivery.Held) delivery.Held {
|
||||||
if fresh != nil {
|
if fresh != nil {
|
||||||
@@ -245,6 +404,8 @@ func (p *passer) heldCondition(ctx context.Context, module, fp string, fresh *de
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
p.said[c.Key+".raised"] = true
|
p.said[c.Key+".raised"] = true
|
||||||
|
delete(p.said, c.Key+".lifted") // a hold raised again and lifted again is said lifted again
|
||||||
|
delete(p.said, c.Key+".cleared") // and released again is cleared again
|
||||||
}
|
}
|
||||||
|
|
||||||
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
|
// restoreAsk writes, once, the ask about a module holding data whose held fingerprint failed its health check on node.
|
||||||
@@ -258,11 +419,21 @@ func (p *passer) restoreAsk(ctx context.Context, module, node, fp string) {
|
|||||||
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||||
}
|
}
|
||||||
|
|
||||||
// answeredWith says whether the teller wrote an answer to the ask at key and the option chosen is option. The answer is the
|
// answeredWith says whether the teller wrote an answer to the ask at key as it stands now and the option chosen is option. The
|
||||||
// operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19).
|
// answer is the operator's, given in the view or on the phone, and stays under its key (ADR 0539 §14, §16, §19); it counts only
|
||||||
|
// for the asking it answered (its warrant names that asking's id), so an answer to an ask taken away, or to an earlier hold of
|
||||||
|
// the same fingerprint, does not carry over to a later one (issue 1906).
|
||||||
func (p *passer) answeredWith(key, option string) bool {
|
func (p *passer) answeredWith(key, option string) bool {
|
||||||
a, ok := p.w.answer(key)
|
a, ok := p.w.answer(key)
|
||||||
return ok && a.Option == option
|
if !ok || a.Option != option {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
ask, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
id, _, _ := strings.Cut(a.Warrant, "@")
|
||||||
|
return id == ask.Ask.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
// clearHeld clears, once, the condition of a hold the operator released.
|
// clearHeld clears, once, the condition of a hold the operator released.
|
||||||
|
|||||||
@@ -0,0 +1,472 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||||
|
|
||||||
|
"mesh-delivery-assigner/internal/assign"
|
||||||
|
)
|
||||||
|
|
||||||
|
// memKV is the part of a bucket a pass writes: puts and deletes.
|
||||||
|
type memKV struct {
|
||||||
|
jetstream.KeyValue
|
||||||
|
values map[string][]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||||
|
m.values[key] = value
|
||||||
|
return uint64(len(m.values)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||||
|
delete(m.values, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newPasser is a pass on a fresh world, as after a start, writing to the buckets given, at the clock given.
|
||||||
|
func newPasser(kvs buckets, at time.Time) *passer {
|
||||||
|
return &passer{kvs: kvs, w: newWorld(), logf: func(string, ...any) {}, said: map[string]bool{}, answeredAt: map[string]uint64{},
|
||||||
|
now: func() time.Time { return at }}
|
||||||
|
}
|
||||||
|
|
||||||
|
var fpTest = delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||||
|
|
||||||
|
// 1906/H2: given a hold lifted by itself, when the assigner is started again and the same fingerprint is held again, then the
|
||||||
|
// hold ask is written under another asking, with an id of its own, so the teller raises it (the router drops a second ask under
|
||||||
|
// an id it keeps).
|
||||||
|
func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||||
|
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||||
|
kvs := buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
||||||
|
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||||
|
held := &delivery.Held{Node: "ace", Words: "health failed"}
|
||||||
|
first := newPasser(kvs, time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC))
|
||||||
|
first.holdAsk(context.Background(), "web", fpTest, held)
|
||||||
|
a1, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
first.lift(context.Background(), "web", fpTest)
|
||||||
|
if _, there := asksKV.values[key]; there {
|
||||||
|
t.Fatal("the lift left the hold ask")
|
||||||
|
}
|
||||||
|
again := newPasser(kvs, time.Date(2026, 10, 25, 2, 0, 0, 0, time.UTC)) // started again: it remembers nothing
|
||||||
|
again.holdAsk(context.Background(), "web", fpTest, held)
|
||||||
|
a2, err := delivery.DecodeAssignerAsk(key, asksKV.values[key])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a2.Ask.ID == a1.Ask.ID {
|
||||||
|
t.Errorf("the ask written again after a lift and a restart has the id %s of the first", a1.Ask.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
|
||||||
|
// taken away, is not taken for the hold that stands now.
|
||||||
|
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||||
|
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||||
|
p := newPasser(buckets{}, time.Now())
|
||||||
|
put := func(asking int) string {
|
||||||
|
a := assign.HoldAsk(key, "web", fpTest, "ace", "health failed", asking, time.Now())
|
||||||
|
v, err := delivery.EncodeAssignerAsk(key, a)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p.w.put(delivery.AssignerAsksBucket, key, v)
|
||||||
|
return a.Ask.ID
|
||||||
|
}
|
||||||
|
first := put(100)
|
||||||
|
v, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: first + "@2026-10-25T01:00:00Z", Words: "the operator"})
|
||||||
|
p.w.put(delivery.AnswersBucket, key, v)
|
||||||
|
if !p.answeredWith(key, "release") {
|
||||||
|
t.Fatal("the answer to the ask that stands is not taken")
|
||||||
|
}
|
||||||
|
if p.answeredWith(key, "keep") {
|
||||||
|
t.Error("an answer was taken for an option not chosen")
|
||||||
|
}
|
||||||
|
put(200) // the same fingerprint held again, after a lift
|
||||||
|
if p.answeredWith(key, "release") {
|
||||||
|
t.Error("the answer to an earlier hold carried over to a later one")
|
||||||
|
}
|
||||||
|
p.w.put(delivery.AssignerAsksBucket, key, nil) // the ask taken away
|
||||||
|
if p.answeredWith(key, "release") {
|
||||||
|
t.Error("an answer to an ask taken away was taken")
|
||||||
|
}
|
||||||
|
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
|
||||||
|
}
|
||||||
|
|
||||||
|
// memEntry is a value a memKV holds, as a bucket's Get answers it.
|
||||||
|
type memEntry struct {
|
||||||
|
jetstream.KeyValueEntry
|
||||||
|
value []byte
|
||||||
|
revision uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e memEntry) Value() []byte { return e.value }
|
||||||
|
func (e memEntry) Revision() uint64 { return e.revision }
|
||||||
|
func (m *memKV) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) {
|
||||||
|
v, ok := m.values[key]
|
||||||
|
if !ok {
|
||||||
|
return nil, jetstream.ErrKeyNotFound
|
||||||
|
}
|
||||||
|
return memEntry{value: v, revision: uint64(len(v))}, nil
|
||||||
|
}
|
||||||
|
func (m *memKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||||
|
if uint64(len(m.values[key])) != revision {
|
||||||
|
return 0, errors.New("wrong last sequence")
|
||||||
|
}
|
||||||
|
m.values[key] = value
|
||||||
|
return uint64(len(value)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// memStore is the runtime's states for the put's tools, over the same memKVs a pass writes.
|
||||||
|
type memStore map[string]*memKV
|
||||||
|
|
||||||
|
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||||
|
v, ok := s[state].values[key]
|
||||||
|
return v, uint64(len(v)), ok, nil
|
||||||
|
}
|
||||||
|
func (s memStore) put(state, key string, value []byte) error {
|
||||||
|
s[state].values[key] = value
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
func (s memStore) keys(state string) ([]string, error) {
|
||||||
|
var out []string
|
||||||
|
for k := range s[state].values {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fakeBus is the assigner's process account in a test: every call recorded, each answered at once.
|
||||||
|
type fakeBus struct {
|
||||||
|
calls chan string
|
||||||
|
events []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *fakeBus) Call(key string, body any) (json.RawMessage, error) {
|
||||||
|
b.calls <- fmt.Sprintf("%s %v", key, body)
|
||||||
|
return json.RawMessage(`{"ok":true,"output":""}`), nil
|
||||||
|
}
|
||||||
|
func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||||
|
return b.Call(node+"/"+key, body)
|
||||||
|
}
|
||||||
|
func (b *fakeBus) Emit(eventType string, body any) error {
|
||||||
|
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0546, the first put (2026-10-25, want.novox.nats): the tool writes the put; a pass waits for the composition and grant
|
||||||
|
// the controller's take-in writes; once they are there it asks the operator with a yes that approves; only on that warrant does it have
|
||||||
|
// the streams snapshotted, then writes the want held and records the warrant; the have running it healthy holds it; a release
|
||||||
|
// gives the want back to the rules. Each step is said as the event put.
|
||||||
|
func putOfTheBus(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
||||||
|
fp := delivery.SourcePrefix2 + "fd497ab8" + strings.Repeat("0", 56)
|
||||||
|
digest := "sha256:" + strings.Repeat("d", 64)
|
||||||
|
mem := func() *memKV { return &memKV{values: map[string][]byte{}} }
|
||||||
|
puts, asksKV, wantsKV := mem(), mem(), mem()
|
||||||
|
kvs := buckets{delivery.AssignerPutsBucket: puts, delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}
|
||||||
|
p := newPasser(kvs, now)
|
||||||
|
bus := &fakeBus{calls: make(chan string, 8)}
|
||||||
|
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
|
||||||
|
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
|
||||||
|
p.acts.snapshot = func(context.Context, string, string) (string, error) {
|
||||||
|
return "novox's restore point of nats (streams) taken 22:01", nil
|
||||||
|
}
|
||||||
|
// The world: nats assigned to novox, built at fp, the bus by its manifest, novox running it from before the switch.
|
||||||
|
world := func(bucket, key string, v any) {
|
||||||
|
var data []byte
|
||||||
|
switch x := v.(type) {
|
||||||
|
case []byte:
|
||||||
|
data = x
|
||||||
|
default:
|
||||||
|
data, _ = json.Marshal(x)
|
||||||
|
}
|
||||||
|
p.w.put(bucket, key, data)
|
||||||
|
}
|
||||||
|
world(provider.AssignmentsBucket, "novox.nats", map[string]string{"node": "novox", "module": "nats"})
|
||||||
|
mdigest := strings.Repeat("e", 64)
|
||||||
|
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: mdigest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("c", 40)})
|
||||||
|
world(delivery.ModulesBucket, "nats", mod)
|
||||||
|
p.w.manifests[mdigest] = parsed{module: "nats", manifest: catalogue.Manifest{Module: "nats", Provides: []catalogue.Offer{{Name: "mesh-bus"}}}}
|
||||||
|
bkey, _ := delivery.BuilderBuildKey("nats", fp)
|
||||||
|
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: mdigest})
|
||||||
|
world(delivery.BuilderBuildsBucket, bkey, built)
|
||||||
|
hkey, _ := delivery.HaveKey("novox", "nats")
|
||||||
|
before, _ := delivery.EncodeHave(delivery.Have{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64), Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||||
|
world(delivery.HaveBucket, hkey, before)
|
||||||
|
|
||||||
|
builds := mem()
|
||||||
|
builds.values[bkey] = built
|
||||||
|
modules := mem()
|
||||||
|
modules.values["nats"] = mod
|
||||||
|
s := newPutTools(memStore{putsState: puts, delivery.BuilderModule + ".builds": builds, delivery.IndexerModule + ".modules": modules})
|
||||||
|
s.now = func() time.Time { return now }
|
||||||
|
was := answerWithin
|
||||||
|
answerWithin = 0
|
||||||
|
defer func() { answerWithin = was }()
|
||||||
|
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": "fd497ab8ffff", "why": "x"}, "shanks.node-tools"); err == nil {
|
||||||
|
t.Error("a prefix of a fingerprint the builder never built was taken")
|
||||||
|
}
|
||||||
|
if got, err := s.version("nats", "fd497ab80000"); err != nil || got != fp {
|
||||||
|
t.Errorf("the prefix of the fingerprint built reads %q, %v", got, err)
|
||||||
|
}
|
||||||
|
if got, err := s.version("nats", "newest"); err != nil || got != fp {
|
||||||
|
t.Errorf("newest reads %q, %v", got, err)
|
||||||
|
}
|
||||||
|
ans, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "the bus's user list"}, "shanks.node-tools")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if a := ans.(putAnswer); a.Step != delivery.PutAsked || !a.Hold || !strings.Contains(a.Follow, "mesh_wait") {
|
||||||
|
t.Fatalf("put answered %+v", a)
|
||||||
|
}
|
||||||
|
pkey, _ := delivery.PutKey("novox", "nats")
|
||||||
|
step := func() delivery.Put {
|
||||||
|
t.Helper()
|
||||||
|
world(delivery.AssignerPutsBucket, pkey, puts.values[pkey])
|
||||||
|
p.pass(ctx)
|
||||||
|
got, err := delivery.DecodePut(puts.values[pkey])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return got
|
||||||
|
}
|
||||||
|
if got := step(); got.Step != delivery.PutCompose || !strings.Contains(got.Words, "no composition") {
|
||||||
|
t.Fatalf("the first pass left the put at %s: %s", got.Step, got.Words)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case call := <-bus.calls:
|
||||||
|
t.Errorf("a put asked %q: it reads the compositions and asks nobody for one", call)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
ckey, _ := delivery.ComposedKey("novox", "nats", fp)
|
||||||
|
cd, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: digest, Judgement: delivery.JudgedCannot, Words: "the have names no composition"})
|
||||||
|
world(delivery.ControllerComposedBucket, ckey, cd)
|
||||||
|
gkey, _ := delivery.ControllerGrantKey("nats", fp)
|
||||||
|
g, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||||
|
world(delivery.ControllerGrantsBucket, gkey, g)
|
||||||
|
got := step()
|
||||||
|
akey, _ := delivery.RecordAskKey("novox", "nats", digest)
|
||||||
|
if got.Step != delivery.PutPerson || got.Ask != akey {
|
||||||
|
t.Fatalf("composed, the put stands at %s (%s), ask %q", got.Step, got.Words, got.Ask)
|
||||||
|
}
|
||||||
|
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||||
|
if err != nil || a.Ask.Options[0].Level != asks.Approve {
|
||||||
|
t.Fatalf("the ask is %+v, %v; want a yes that approves", a, err)
|
||||||
|
}
|
||||||
|
if _, there := wantsKV.values["want.novox.nats"]; there {
|
||||||
|
t.Fatal("the bus's want was written before the operator's warrant")
|
||||||
|
}
|
||||||
|
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||||
|
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-25T22:01:00Z", Words: "the operator, verified"})
|
||||||
|
world(delivery.AnswersBucket, akey, yes)
|
||||||
|
if got := step(); got.Step != delivery.PutSnapshot {
|
||||||
|
t.Fatalf("on the warrant the put stands at %s: %s", got.Step, got.Words)
|
||||||
|
}
|
||||||
|
<-woke
|
||||||
|
got = step()
|
||||||
|
w, err := delivery.DecodeWant(wantsKV.values["want.novox.nats"])
|
||||||
|
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
|
||||||
|
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
|
||||||
|
}
|
||||||
|
if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) {
|
||||||
|
t.Errorf("the warrant was recorded as %q", call)
|
||||||
|
}
|
||||||
|
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
|
||||||
|
running, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||||
|
world(delivery.HaveBucket, hkey, running)
|
||||||
|
if got := step(); got.Step != delivery.PutHolding {
|
||||||
|
t.Fatalf("running healthy, the put stands at %s: %s", got.Step, got.Words)
|
||||||
|
}
|
||||||
|
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "done"}, "shanks.node-tools"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := step(); got.Step != delivery.PutReleased {
|
||||||
|
t.Fatalf("released, the put stands at %s", got.Step)
|
||||||
|
}
|
||||||
|
listed, _ := s.doPuts(map[string]any{"module": "nats"})
|
||||||
|
if l := listed.(map[string]any)["puts"].([]putAnswer); len(l) != 1 || l[0].Step != delivery.PutReleased || len(l[0].Steps) < 6 {
|
||||||
|
t.Errorf("puts lists %+v", l)
|
||||||
|
}
|
||||||
|
if len(bus.events) < 5 || !strings.Contains(strings.Join(bus.events, "\n"), "holding") {
|
||||||
|
t.Errorf("the steps said as events: %v", bus.events)
|
||||||
|
}
|
||||||
|
// The same bus put again at the same composition after the release is its own asking: its ask is written again under an
|
||||||
|
// id of its own, and the earlier yes, still at the answers' key, does not count for it (ADR 0546 clause 8).
|
||||||
|
world(delivery.HaveBucket, hkey, before)
|
||||||
|
s.now = func() time.Time { return now.Add(time.Minute) }
|
||||||
|
if _, err := s.doPut(map[string]any{"node": "novox", "module": "nats", "version": fp, "hold": "true", "why": "again"}, "shanks.node-tools"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
again := step()
|
||||||
|
a2, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
|
||||||
|
if again.Step != delivery.PutPerson || err != nil || a2.Ask.ID == a.Ask.ID {
|
||||||
|
t.Fatalf("put again, it stands at %s (%s); its ask %q, %v; the first ask was %q", again.Step, again.Words, a2.Ask.ID, err, a.Ask.ID)
|
||||||
|
}
|
||||||
|
world(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
|
||||||
|
yes2, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a2.Ask.ID + "@2026-10-25T22:03:00Z", Words: "the operator, verified"})
|
||||||
|
world(delivery.AnswersBucket, akey, yes2)
|
||||||
|
if got := step(); got.Step != delivery.PutSnapshot {
|
||||||
|
t.Fatalf("on the second warrant the put stands at %s: %s", got.Step, got.Words)
|
||||||
|
}
|
||||||
|
<-woke
|
||||||
|
// Released between the pass's read and its write: the pass writes no want for the put that no longer stands.
|
||||||
|
oldWant, _ := delivery.EncodeWant(delivery.Want{FP: delivery.SourcePrefix2 + strings.Repeat("a", 64)})
|
||||||
|
wantsKV.values["want.novox.nats"] = oldWant
|
||||||
|
world(delivery.AssignerWantsBucket, "want.novox.nats", oldWant)
|
||||||
|
if _, err := s.doRelease(map[string]any{"node": "novox", "module": "nats", "why": "changed my mind"}, "shanks.node-tools"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
p.pass(ctx)
|
||||||
|
if string(wantsKV.values["want.novox.nats"]) != string(oldWant) {
|
||||||
|
t.Errorf("a want was written for a put released meanwhile: %s", wantsKV.values["want.novox.nats"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC21: given a module on neither list on a node, when passes follow, then the assigner raises its condition once, says it
|
||||||
|
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
|
||||||
|
// the journal, so it is loud with no operator channel.
|
||||||
|
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||||
|
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||||
|
var said []asks.Condition
|
||||||
|
failing := true
|
||||||
|
p.say = func(c asks.Condition) error {
|
||||||
|
if failing {
|
||||||
|
failing = false
|
||||||
|
return errors.New("the channel is away")
|
||||||
|
}
|
||||||
|
said = append(said, c)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var logged []string
|
||||||
|
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
|
||||||
|
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
|
||||||
|
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
|
||||||
|
p.sayNeither(one, nil) // the say fails
|
||||||
|
p.sayNeither(one, nil)
|
||||||
|
p.sayNeither(one, nil)
|
||||||
|
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
|
||||||
|
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
|
||||||
|
}
|
||||||
|
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
|
||||||
|
t.Errorf("the journal does not name the module on neither list: %q", logged)
|
||||||
|
}
|
||||||
|
p.sayNeither(map[string]neitherOne{}, nil)
|
||||||
|
p.sayNeither(map[string]neitherOne{}, nil)
|
||||||
|
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
|
||||||
|
t.Errorf("said = %+v, want the condition cleared once under its key", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
|
||||||
|
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||||
|
got := map[string]neitherOne{}
|
||||||
|
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
|
||||||
|
gatherNeither(got, "rofi", assign.Outcome{})
|
||||||
|
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
|
||||||
|
t.Errorf("gathered = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
|
||||||
|
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
|
||||||
|
// every module that is not held. Clearing a condition that is not open is quiet at the router.
|
||||||
|
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
|
||||||
|
wantsKV := &memKV{values: map[string][]byte{}}
|
||||||
|
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||||
|
var said []asks.Condition
|
||||||
|
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
|
||||||
|
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||||
|
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||||
|
p.w.put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
|
||||||
|
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||||
|
p.w.put(delivery.PresenceBucket, "g14", on)
|
||||||
|
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||||
|
p.w.put(delivery.HaveBucket, "g14.web", have)
|
||||||
|
key, _ := delivery.WantKey("g14", "web")
|
||||||
|
want, _ := delivery.EncodeWant(delivery.Want{FP: fpTest, Composed: "sha256:" + strings.Repeat("c", 64)})
|
||||||
|
p.w.put(delivery.AssignerWantsBucket, key, want)
|
||||||
|
p.wants(context.Background())
|
||||||
|
cleared := map[string]bool{}
|
||||||
|
for _, c := range said {
|
||||||
|
if c.State != asks.StateCleared {
|
||||||
|
t.Errorf("the first pass raised %+v", c)
|
||||||
|
}
|
||||||
|
cleared[c.Key] = true
|
||||||
|
}
|
||||||
|
neither := assign.NeitherCondition("g14", "web", assign.Holding{}, false).Key
|
||||||
|
held := assign.HeldCondition("web", fpTest, "", "", false, false).Key
|
||||||
|
if !cleared[neither] || !cleared[held] {
|
||||||
|
t.Errorf("cleared = %v, want %s and %s", cleared, neither, held)
|
||||||
|
}
|
||||||
|
n := len(said)
|
||||||
|
p.wants(context.Background())
|
||||||
|
if len(said) != n {
|
||||||
|
t.Errorf("a second pass said again: %+v", said[n:])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
|
||||||
|
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
|
||||||
|
// raised (the first pass after a start), each once.
|
||||||
|
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
|
||||||
|
setup := func() (*passer, *[]asks.Condition) {
|
||||||
|
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||||
|
said := &[]asks.Condition{}
|
||||||
|
p.say = func(c asks.Condition) error { *said = append(*said, c); return nil }
|
||||||
|
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/resolved", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||||
|
p.w.put(delivery.ModulesBucket, "systemd-resolved", mod)
|
||||||
|
p.w.put(provider.AssignmentsBucket, "shanks.systemd-resolved", []byte(`{"node":"shanks","module":"systemd-resolved","build":"b-1"}`))
|
||||||
|
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||||
|
p.w.put(delivery.PresenceBucket, "shanks", on)
|
||||||
|
have, _ := delivery.EncodeHave(delivery.Have{FP: fpTest, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed})
|
||||||
|
p.w.put(delivery.HaveBucket, "shanks.systemd-resolved", have)
|
||||||
|
hkey, _ := delivery.HeldKey("systemd-resolved", fpTest)
|
||||||
|
held, _ := delivery.EncodeHeld(delivery.Held{Node: "shanks", Words: "health failed"})
|
||||||
|
p.w.put(delivery.AssignerWantsBucket, hkey, held)
|
||||||
|
return p, said
|
||||||
|
}
|
||||||
|
key := assign.NeitherCondition("shanks", "systemd-resolved", assign.Holding{}, false).Key
|
||||||
|
neithers := func(said []asks.Condition) (raised, cleared int) {
|
||||||
|
for _, c := range said {
|
||||||
|
if c.Key != key {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c.State == asks.StateRaised {
|
||||||
|
raised++
|
||||||
|
} else {
|
||||||
|
cleared++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// a standing neither this process raised
|
||||||
|
p, said := setup()
|
||||||
|
p.neither = map[string]neitherOne{"shanks.systemd-resolved": {node: "shanks", module: "systemd-resolved", holding: assign.Holding{Rule: delivery.RuleNotHeld}}}
|
||||||
|
p.wants(context.Background())
|
||||||
|
p.wants(context.Background())
|
||||||
|
if r, c := neithers(*said); r != 0 || c != 1 {
|
||||||
|
t.Errorf("standing in this process: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||||
|
}
|
||||||
|
// one a previous process raised: the first pass after a start clears it
|
||||||
|
p, said = setup()
|
||||||
|
p.wants(context.Background())
|
||||||
|
p.wants(context.Background())
|
||||||
|
if r, c := neithers(*said); r != 0 || c != 1 {
|
||||||
|
t.Errorf("after a start: %d raised, %d cleared, want 0 and 1: %+v", r, c, *said)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,449 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-assigner/internal/assign"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The puts (novox/hq ADR 0546): the tool `assigner_put` writes the key `<node>.<module>` of the assigner's puts, and each pass
|
||||||
|
// takes every put that has not ended one step further (assign.StepPut), doing the acts the step asks for: the put's ask, the
|
||||||
|
// bus's snapshot, the warranted record. Each act's outcome is heard back as a change of the world (the composition, the grant,
|
||||||
|
// the answer) or, for the calls, as a wake once they answer; nothing polls but the snapshot's read-back of the backup holder,
|
||||||
|
// as the controller's bus step reads it (ADR 0235), bounded. Every step is said as the event mesh-delivery-assigner.put, so a
|
||||||
|
// caller waits on it.
|
||||||
|
|
||||||
|
// PutEvent is the event a put's every step is said as: mesh.mod.mesh-delivery-assigner.event.put.
|
||||||
|
const PutEvent = "put"
|
||||||
|
|
||||||
|
// caller is what the puts need of the bus as the assigner's own process account: a call of a seat's verb, on one node or on
|
||||||
|
// the seat held once, and an event said. process.Conn, or a test's.
|
||||||
|
type caller interface {
|
||||||
|
Call(key string, body any) (json.RawMessage, error)
|
||||||
|
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||||
|
Emit(eventType string, body any) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// putActs keeps, by put (its key and when it was asked), what the acts of a put came to while this process runs: a restart
|
||||||
|
// takes the snapshot again, without harm.
|
||||||
|
type putActs struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
bus caller
|
||||||
|
wake func()
|
||||||
|
logf func(string, ...any)
|
||||||
|
snapping map[string]bool
|
||||||
|
snapped map[string]string
|
||||||
|
snapNo map[string]string
|
||||||
|
warranted map[string]bool
|
||||||
|
warrantNo map[string]string
|
||||||
|
said map[string]string
|
||||||
|
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
|
||||||
|
snapshot func(ctx context.Context, node, module string) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
|
||||||
|
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
|
||||||
|
said: map[string]string{}}
|
||||||
|
a.snapshot = a.snapshotTheBus
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
|
||||||
|
// identity names one put: its key and when it was asked, so a put asked again under the key is another.
|
||||||
|
func identity(key string, p delivery.Put) string {
|
||||||
|
return key + "@" + p.At.UTC().Format(time.RFC3339Nano)
|
||||||
|
}
|
||||||
|
|
||||||
|
// putsByModule is every put the assigner holds, by module then node.
|
||||||
|
func (p *passer) putsByModule() map[string]map[string]delivery.Put {
|
||||||
|
out := map[string]map[string]delivery.Put{}
|
||||||
|
for key, value := range p.w.read(delivery.AssignerPutsBucket) {
|
||||||
|
node, module, err := delivery.ParsePutKey(key)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
put, err := delivery.DecodePut(value)
|
||||||
|
if err != nil {
|
||||||
|
p.logf("the put %s cannot be read, and is left: %v", key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if out[module] == nil {
|
||||||
|
out[module] = map[string]delivery.Put{}
|
||||||
|
}
|
||||||
|
out[module][node] = put
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// puts takes every put that has not ended one step further.
|
||||||
|
func (p *passer) puts(ctx context.Context) {
|
||||||
|
if p.acts == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
all := p.putsByModule()
|
||||||
|
if len(all) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
assigned, _ := p.w.assigned()
|
||||||
|
trunk := p.w.trunk()
|
||||||
|
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||||
|
wanted := p.w.wantsOf()
|
||||||
|
modules := make([]string, 0, len(all))
|
||||||
|
for m := range all {
|
||||||
|
modules = append(modules, m)
|
||||||
|
}
|
||||||
|
sort.Strings(modules)
|
||||||
|
for _, module := range modules {
|
||||||
|
var manifest catalogue.Manifest
|
||||||
|
if mod, ok := trunk[module]; ok && mod.Manifest != "" {
|
||||||
|
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
|
||||||
|
manifest = got.manifest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for node, put := range all[module] {
|
||||||
|
if delivery.PutEnded(put.Step) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key, _ := delivery.PutKey(node, module)
|
||||||
|
id := identity(key, put)
|
||||||
|
world := p.putWorld(node, module, put, manifest, assigned[node])
|
||||||
|
world.Snapshot, world.Snapshotting, world.SnapshotFailed = p.acts.state(id)
|
||||||
|
next := assign.StepPut(world)
|
||||||
|
// A held bus whose user list the rules moved on (ADR 0539 §7): the put follows the composition its want names.
|
||||||
|
if w, ok := wanted[module][node]; ok && w.Put && w.FP == next.Put.FP && w.Composed != "" && w.Composed != next.Put.Composed &&
|
||||||
|
(next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) {
|
||||||
|
if d, _ := p.composedFor(node, module, w.FP); d != nil && d.UserListOnly && d.Digest == w.Composed {
|
||||||
|
next.Put = next.Put.Said(world.Now, next.Put.Step, fmt.Sprintf("the bus's user list changed: the want names composition %s", shortHex(w.Composed)))
|
||||||
|
next.Put.Composed = w.Composed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The bus's want is written only for the act the operator's yes bound (ADR 0546 clause 8).
|
||||||
|
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted && world.Bus && next.Put.Ask != "" {
|
||||||
|
if err := p.performs(next.Put.Ask, assign.PutAct(node, module, next.Put.FP, next.Put.Composed)); err != nil {
|
||||||
|
next = assign.PutNext{Put: next.Put.Said(world.Now, delivery.PutRefused, "the bus's want is not written: "+err.Error())}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if failed := p.acts.warrantFailed(id); failed != "" && (next.Put.Step == delivery.PutWanted || next.Put.Step == delivery.PutHolding) &&
|
||||||
|
!strings.Contains(next.Put.Words, failed) {
|
||||||
|
next.Put = next.Put.Said(world.Now, next.Put.Step, next.Put.Words+"; "+failed)
|
||||||
|
}
|
||||||
|
p.actOn(ctx, node, module, id, world, next)
|
||||||
|
written := samePut(put, next.Put) || p.writePut(ctx, key, put, next.Put)
|
||||||
|
if !written {
|
||||||
|
continue // released or put again meanwhile: nothing is done for the put that no longer stands
|
||||||
|
}
|
||||||
|
if next.Put.Step == delivery.PutWanted && put.Step != delivery.PutWanted {
|
||||||
|
// The want is written in this pass too, before the next one would: Module.Puts writes the same.
|
||||||
|
if wkey, err := delivery.WantKey(node, module); err == nil {
|
||||||
|
want := assign.PutWant(next.Put)
|
||||||
|
if before, ok := wanted[module][node]; !ok || !delivery.SameWant(normalWant(before), normalWant(want)) {
|
||||||
|
p.put(ctx, delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if next.Warranted {
|
||||||
|
p.acts.warrant(id, delivery.AskingID(next.Put.Ask, assign.PutAsking(next.Put)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalWant(w delivery.Want) delivery.Want {
|
||||||
|
w.V = 0
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
// putWorld is what one step of a put reads from the world.
|
||||||
|
func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalogue.Manifest, assignedHere []string) assign.PutWorld {
|
||||||
|
w := assign.PutWorld{Node: node, Module: module, Put: put, Now: p.clock(), Bus: catalogue.ProvidesBus(manifest), Newest: p.w.trunk()[module].FP2}
|
||||||
|
for _, m := range assignedHere {
|
||||||
|
if m == module {
|
||||||
|
w.Assigned = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if bkey, err := delivery.BuilderBuildKey(module, put.FP); err == nil {
|
||||||
|
if raw := p.w.value(delivery.BuilderBuildsBucket, bkey); raw != nil {
|
||||||
|
if b, err := delivery.DecodeBuilderBuild(raw); err == nil {
|
||||||
|
w.Build, w.BuildWords = b.State, b.Error
|
||||||
|
} else {
|
||||||
|
w.Build = "unreadable: " + err.Error()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
|
||||||
|
if d := w.Composed; d != nil {
|
||||||
|
policy := d.Policy
|
||||||
|
if policy == "" {
|
||||||
|
policy, _, _ = catalogue.DerivedUpgrade(manifest)
|
||||||
|
}
|
||||||
|
w.Recorded = policy == catalogue.PolicyRecord
|
||||||
|
if akey, err := delivery.RecordAskKey(node, module, d.Digest); err == nil && put.Ask == akey {
|
||||||
|
// Only this put's own ask counts: an ask (and its answer) of an earlier put under the same key is another asking.
|
||||||
|
if a, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey)); err == nil && a.Asking == assign.PutAsking(put) {
|
||||||
|
switch {
|
||||||
|
case p.answeredWith(akey, assign.OptionYes):
|
||||||
|
w.Answer = assign.OptionYes
|
||||||
|
case p.answeredWith(akey, assign.OptionNo):
|
||||||
|
w.Answer = assign.OptionNo
|
||||||
|
default:
|
||||||
|
w.AskExpired = !a.Ask.Expires.IsZero() && w.Now.After(a.Ask.Expires)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if hkey, err := delivery.HaveKey(node, module); err == nil {
|
||||||
|
if raw := p.w.value(delivery.HaveBucket, hkey); raw != nil {
|
||||||
|
if h, err := delivery.DecodeHave(raw); err == nil {
|
||||||
|
w.Have = &h
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
|
||||||
|
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
|
||||||
|
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
|
||||||
|
// from their new owner and nothing else changes.
|
||||||
|
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
|
||||||
|
var out *delivery.ComposedDigest
|
||||||
|
if d, ok := p.w.composed(node, module, fp); ok {
|
||||||
|
out = &d
|
||||||
|
}
|
||||||
|
granted := false
|
||||||
|
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
|
||||||
|
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
|
||||||
|
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
|
||||||
|
granted = g.NamesNode(node, module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, granted
|
||||||
|
}
|
||||||
|
|
||||||
|
// actOn does the acts a step asks for.
|
||||||
|
func (p *passer) actOn(ctx context.Context, node, module, id string, w assign.PutWorld, next assign.PutNext) {
|
||||||
|
if next.Put.Step == delivery.PutPerson && next.Put.Ask != "" && w.Composed != nil {
|
||||||
|
// The put's own ask, written again over one an earlier put left under the key (its asking differs), so the teller
|
||||||
|
// raises it anew under its own id.
|
||||||
|
akey := next.Put.Ask
|
||||||
|
current, err := delivery.DecodeAssignerAsk(akey, p.w.value(delivery.AssignerAsksBucket, akey))
|
||||||
|
if err != nil || current.Asking != assign.PutAsking(next.Put) {
|
||||||
|
a, err := assign.PutAsk(akey, node, module, next.Put, *w.Composed, w.Bus, w.Now)
|
||||||
|
if err != nil {
|
||||||
|
p.logf("the ask about putting %s on %s is not written: %v", module, node, err)
|
||||||
|
} else {
|
||||||
|
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if next.Snapshot {
|
||||||
|
p.acts.snap(ctx, id, node, module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// performs checks that the act about to be performed is the one the yes of the ask at key bound (asks.Option.Performs).
|
||||||
|
func (p *passer) performs(key string, act asks.Act) error {
|
||||||
|
a, err := delivery.DecodeAssignerAsk(key, p.w.value(delivery.AssignerAsksBucket, key))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the ask %s cannot be read: %w", key, err)
|
||||||
|
}
|
||||||
|
for _, o := range a.Ask.Options {
|
||||||
|
if o.ID == assign.OptionYes {
|
||||||
|
return o.Performs(act)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("the ask %s offers no yes", key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// samePut says two puts say the same: their step, words and what they found.
|
||||||
|
func samePut(a, b delivery.Put) bool {
|
||||||
|
return a.Step == b.Step && a.Words == b.Words && a.Composed == b.Composed && a.Ask == b.Ask && a.Snapshot == b.Snapshot
|
||||||
|
}
|
||||||
|
|
||||||
|
// writePut writes a put's next value over the one the pass read, by compare-and-set, unless the tool has put again or
|
||||||
|
// released it since; and says it as the event put. It says whether it wrote.
|
||||||
|
func (p *passer) writePut(ctx context.Context, key string, read, next delivery.Put) bool {
|
||||||
|
kv := p.kvs[delivery.AssignerPutsBucket]
|
||||||
|
if kv == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
p.logf("the put %s cannot be read again, and is taken further at the next pass: %v", key, err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
now, err := delivery.DecodePut(e.Value())
|
||||||
|
if err != nil || !now.At.Equal(read.At) || now.Step != read.Step {
|
||||||
|
return false // asked again or released meanwhile: the next pass reads that
|
||||||
|
}
|
||||||
|
data, err := delivery.EncodePut(next)
|
||||||
|
if err != nil {
|
||||||
|
p.logf("the put %s is not written: %v", key, err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, err := kv.Update(ctx, key, data, e.Revision()); err != nil {
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
p.logf("the put %s could not be written, and is taken further at the next pass: %v", key, err)
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
p.w.put(delivery.AssignerPutsBucket, key, data)
|
||||||
|
p.logf("put %s: %s: %s", key, next.Step, next.Words)
|
||||||
|
p.acts.say(key, next)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// state is what this process knows of a put's acts.
|
||||||
|
func (a *putActs) state(id string) (snapshot string, snapping bool, snapNo string) {
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
return a.snapped[id], a.snapping[id], a.snapNo[id]
|
||||||
|
}
|
||||||
|
|
||||||
|
// snap has the bus's snapshot taken once for a put, in the background.
|
||||||
|
func (a *putActs) snap(ctx context.Context, id, node, module string) {
|
||||||
|
a.mu.Lock()
|
||||||
|
if a.snapping[id] || a.snapped[id] != "" || a.snapNo[id] != "" {
|
||||||
|
a.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.snapping[id] = true
|
||||||
|
a.mu.Unlock()
|
||||||
|
go func() {
|
||||||
|
defer a.wake()
|
||||||
|
where, err := a.snapshot(ctx, node, module)
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
a.snapping[id] = false
|
||||||
|
if err != nil {
|
||||||
|
a.snapNo[id] = err.Error()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.snapped[id] = where
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// SnapshotWithin is how long the bus node's backup holder is given to take the bus's snapshot (as the controller's bus step).
|
||||||
|
var SnapshotWithin = 15 * time.Minute
|
||||||
|
|
||||||
|
// snapshotLook is how often the backup holder's record is read back while the snapshot is taken.
|
||||||
|
var snapshotLook = 10 * time.Second
|
||||||
|
|
||||||
|
// backedUp is one module as the backup holder's `backed-up` answers it.
|
||||||
|
type backedUp struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
Data []struct {
|
||||||
|
Item string `json:"item"`
|
||||||
|
LastBackup *time.Time `json:"last_backup"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
} `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// snapshotTheBus asks the bus node's backup holder to back the bus module up now, whose dump is the streams' snapshot
|
||||||
|
// (ADR 0235), and reads its record back until it names a backup newer than the ask: where the snapshot is.
|
||||||
|
func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (string, error) {
|
||||||
|
asked := time.Now()
|
||||||
|
if _, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".now", map[string]any{"module": module}); err != nil {
|
||||||
|
return "", fmt.Errorf("%s's backup holder was not asked to take the bus's snapshot: %w", node, err)
|
||||||
|
}
|
||||||
|
deadline := asked.Add(SnapshotWithin)
|
||||||
|
for {
|
||||||
|
if raw, err := a.bus.CallOn(node, "seat:"+catalogue.BackupSeat+".backed-up", map[string]any{}); err == nil {
|
||||||
|
var modules []backedUp
|
||||||
|
if json.Unmarshal(raw, &modules) == nil {
|
||||||
|
for _, m := range modules {
|
||||||
|
for _, it := range m.Data {
|
||||||
|
if m.Module == module && it.LastBackup != nil && it.LastBackup.After(asked) && it.Error == "" {
|
||||||
|
return fmt.Sprintf("%s's restore point of %s (%s) taken %s", node, module, it.Item, it.LastBackup.UTC().Format(time.RFC3339)), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
return "", fmt.Errorf("%s's backup holder did not say the bus's snapshot was taken within %s", node, SnapshotWithin)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return "", ctx.Err()
|
||||||
|
case <-time.After(snapshotLook):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
|
||||||
|
// router's record of the teller's ask says who chose and how (ADR 0274).
|
||||||
|
func (a *putActs) warrant(id, ask string) {
|
||||||
|
a.mu.Lock()
|
||||||
|
if a.warranted[id] {
|
||||||
|
a.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.warranted[id] = true
|
||||||
|
a.mu.Unlock()
|
||||||
|
go func() {
|
||||||
|
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
|
||||||
|
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
|
||||||
|
a.mu.Lock()
|
||||||
|
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
|
||||||
|
a.mu.Unlock()
|
||||||
|
a.wake()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// warrantFailed says why the put's warrant was not recorded, "" when it was or is not yet known.
|
||||||
|
func (a *putActs) warrantFailed(id string) string {
|
||||||
|
a.mu.Lock()
|
||||||
|
defer a.mu.Unlock()
|
||||||
|
return a.warrantNo[id]
|
||||||
|
}
|
||||||
|
|
||||||
|
// putSaid is the body of the event put.
|
||||||
|
type putSaid struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
FP string `json:"fp"`
|
||||||
|
Hold bool `json:"hold"`
|
||||||
|
Step string `json:"step"`
|
||||||
|
Words string `json:"words"`
|
||||||
|
Composed string `json:"composed,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// say says a put's step as the event put, once per step and words.
|
||||||
|
func (a *putActs) say(key string, p delivery.Put) {
|
||||||
|
node, module, err := delivery.ParsePutKey(key)
|
||||||
|
if err != nil || a.bus == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.mu.Lock()
|
||||||
|
if a.said[key] == p.Step+p.Words {
|
||||||
|
a.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a.said[key] = p.Step + p.Words
|
||||||
|
a.mu.Unlock()
|
||||||
|
if err := a.bus.Emit(PutEvent, putSaid{Node: node, Module: module, FP: p.FP, Hold: p.Hold, Step: p.Step, Words: p.Words, Composed: p.Composed}); err != nil {
|
||||||
|
a.logf("the put %s's step %s is not said as an event: %v", key, p.Step, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// shortHex is a digest's first twelve hex digits.
|
||||||
|
func shortHex(digest string) string {
|
||||||
|
h := strings.TrimPrefix(digest, "sha256:")
|
||||||
|
if len(h) > 12 {
|
||||||
|
return h[:12]
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
}
|
||||||
@@ -0,0 +1,364 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||||
|
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
|
||||||
|
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
|
||||||
|
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
|
||||||
|
|
||||||
|
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||||
|
const putsState = "puts"
|
||||||
|
|
||||||
|
// answerWithin is how long assigner_put waits for the assigner's process to take the put up before it answers what stands.
|
||||||
|
var answerWithin = 20 * time.Second
|
||||||
|
|
||||||
|
// store is what the tools need of the states: the runtime's (stdio.State), or a test's.
|
||||||
|
type store interface {
|
||||||
|
get(state, key string) ([]byte, uint64, bool, error)
|
||||||
|
put(state, key string, value []byte) error
|
||||||
|
keys(state string) ([]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// putTools serves the tools over a store, and keeps the newest value of every put it heard, to wait on.
|
||||||
|
type putTools struct {
|
||||||
|
st store
|
||||||
|
now func() time.Time
|
||||||
|
mu sync.Mutex
|
||||||
|
// heard is every put the watch handed, by key; changed is closed and replaced at each.
|
||||||
|
heard map[string][]byte
|
||||||
|
changed chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPutTools(st store) *putTools {
|
||||||
|
return &putTools{st: st, now: time.Now, heard: map[string][]byte{}, changed: make(chan struct{})}
|
||||||
|
}
|
||||||
|
|
||||||
|
// hear takes a put's change in from the watch.
|
||||||
|
func (s *putTools) hear(key string, value []byte) {
|
||||||
|
s.mu.Lock()
|
||||||
|
s.heard[key] = value
|
||||||
|
close(s.changed)
|
||||||
|
s.changed = make(chan struct{})
|
||||||
|
s.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// fullFP is a fingerprint of the new form, whole; prefix at least its first twelve hex digits.
|
||||||
|
var (
|
||||||
|
fullFP = regexp.MustCompile(`^src2:[0-9a-f]{64}$`)
|
||||||
|
prefix = regexp.MustCompile(`^(src2:)?[0-9a-f]{12,63}$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// version reads the version a put names: newest (the indexer's fingerprint at the trunk), a whole fingerprint, or the start
|
||||||
|
// of one the builder built, which must name one build alone.
|
||||||
|
func (s *putTools) version(module, version string) (string, error) {
|
||||||
|
v := strings.ToLower(strings.TrimSpace(version))
|
||||||
|
switch {
|
||||||
|
case v == "newest":
|
||||||
|
raw, _, found, err := s.st.get(delivery.IndexerModule+".modules", module)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the indexer's modules cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return "", fmt.Errorf("the indexer holds no module %s at the trunk", module)
|
||||||
|
}
|
||||||
|
m, err := delivery.DecodeModule(raw)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the indexer's %s cannot be read: %w", module, err)
|
||||||
|
}
|
||||||
|
if m.FP2 == "" {
|
||||||
|
return "", fmt.Errorf("the indexer holds no fingerprint of the new form for %s: %s", module, m.None2)
|
||||||
|
}
|
||||||
|
return m.FP2, nil
|
||||||
|
case fullFP.MatchString(v):
|
||||||
|
return v, nil
|
||||||
|
case prefix.MatchString(v):
|
||||||
|
start := strings.TrimPrefix(v, delivery.SourcePrefix2)
|
||||||
|
keys, err := s.st.keys(delivery.BuilderModule + ".builds")
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the builder's builds cannot be listed: %w", err)
|
||||||
|
}
|
||||||
|
var found []string
|
||||||
|
for _, k := range keys {
|
||||||
|
m, fp, err := delivery.ParseBuilderBuildKey(k)
|
||||||
|
if err == nil && m == module && strings.HasPrefix(strings.TrimPrefix(fp, delivery.SourcePrefix2), start) {
|
||||||
|
found = append(found, fp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch len(found) {
|
||||||
|
case 0:
|
||||||
|
return "", fmt.Errorf("the builder built no %s whose fingerprint starts %s: give it whole", module, start)
|
||||||
|
case 1:
|
||||||
|
return found[0], nil
|
||||||
|
}
|
||||||
|
sort.Strings(found)
|
||||||
|
return "", fmt.Errorf("%s starts %d fingerprints of %s the builder built (%s): give more of it", start, len(found), module, strings.Join(found, ", "))
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("%q is no version: newest, a fingerprint (src2: and 64 hex digits) or at least its first 12 hex digits", version)
|
||||||
|
}
|
||||||
|
|
||||||
|
// putAnswer is what assigner_put and assigner_puts answer of one put.
|
||||||
|
type putAnswer struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
delivery.Put
|
||||||
|
Follow string `json:"follow,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// doPut is the tool assigner_put.
|
||||||
|
func (s *putTools) doPut(args map[string]any, by string) (any, error) {
|
||||||
|
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||||
|
key, err := delivery.PutKey(node, module)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if why == "" {
|
||||||
|
return nil, errors.New("put says why: it is kept with the put and said in any ask")
|
||||||
|
}
|
||||||
|
hold := false
|
||||||
|
switch arg(args, "hold") {
|
||||||
|
case "true":
|
||||||
|
hold = true
|
||||||
|
case "", "false":
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("hold is true or false, not %q", arg(args, "hold"))
|
||||||
|
}
|
||||||
|
fp, err := s.version(module, arg(args, "version"))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
now := s.now().UTC()
|
||||||
|
words := fmt.Sprintf("asked by %s: %s", by, why)
|
||||||
|
if raw, _, found, err := s.st.get(putsState, key); err != nil {
|
||||||
|
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||||
|
} else if found {
|
||||||
|
if before, err := delivery.DecodePut(raw); err == nil && !delivery.PutEnded(before.Step) {
|
||||||
|
if before.FP == fp && before.Hold == hold {
|
||||||
|
return putAnswer{Node: node, Module: module, Put: before, Follow: followPut(node, module)}, nil
|
||||||
|
}
|
||||||
|
words += fmt.Sprintf("; it replaces the put of %s at %s, which stood at %s", module, shortFP(before.FP), before.Step)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p := delivery.Put{FP: fp, Hold: hold, Why: why, By: by, At: now}.Said(now, delivery.PutAsked, words)
|
||||||
|
data, err := delivery.EncodePut(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := s.st.put(putsState, key, data); err != nil {
|
||||||
|
return nil, fmt.Errorf("the put of %s on %s is not written: %w", module, node, err)
|
||||||
|
}
|
||||||
|
s.hear(key, data)
|
||||||
|
return putAnswer{Node: node, Module: module, Put: s.await(key, p), Follow: followPut(node, module)}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// await is the put at key once the assigner's process took it up past its asking, or as it stands after answerWithin.
|
||||||
|
func (s *putTools) await(key string, p delivery.Put) delivery.Put {
|
||||||
|
deadline := time.After(answerWithin)
|
||||||
|
for {
|
||||||
|
s.mu.Lock()
|
||||||
|
raw, changed := s.heard[key], s.changed
|
||||||
|
s.mu.Unlock()
|
||||||
|
if now, err := delivery.DecodePut(raw); err == nil && now.At.Equal(p.At) {
|
||||||
|
p = now
|
||||||
|
if now.Step != delivery.PutAsked && now.Step != delivery.PutBuild && now.Step != delivery.PutCompose {
|
||||||
|
return p // it waits on a person or the node, or ended
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-changed:
|
||||||
|
case <-deadline:
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// followPut says how a caller follows a put after the answer.
|
||||||
|
func followPut(node, module string) string {
|
||||||
|
return fmt.Sprintf("the put goes on: mesh_wait on mesh-delivery-assigner.put with filter {\"node\": %q, \"module\": %q}, then "+
|
||||||
|
"mesh-delivery-assigner.assigner_puts for where it stands", node, module)
|
||||||
|
}
|
||||||
|
|
||||||
|
// doRelease is the tool assigner_release.
|
||||||
|
func (s *putTools) doRelease(args map[string]any, by string) (any, error) {
|
||||||
|
node, module, why := arg(args, "node"), arg(args, "module"), arg(args, "why")
|
||||||
|
key, err := delivery.PutKey(node, module)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if why == "" {
|
||||||
|
return nil, errors.New("release says why: it is kept with the put")
|
||||||
|
}
|
||||||
|
raw, _, found, err := s.st.get(putsState, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return nil, fmt.Errorf("%s holds no put of %s: nothing to release", node, module)
|
||||||
|
}
|
||||||
|
p, err := delivery.DecodePut(raw)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the put of %s on %s cannot be read: %w", module, node, err)
|
||||||
|
}
|
||||||
|
if delivery.PutEnded(p.Step) {
|
||||||
|
return nil, fmt.Errorf("the put of %s on %s ended already (%s: %s): nothing to release", module, node, p.Step, p.Words)
|
||||||
|
}
|
||||||
|
p = p.Said(s.now(), delivery.PutReleased, fmt.Sprintf("released by %s: %s; the assigner's rules write the want again", by, why))
|
||||||
|
data, err := delivery.EncodePut(p)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := s.st.put(putsState, key, data); err != nil {
|
||||||
|
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
|
||||||
|
}
|
||||||
|
s.hear(key, data)
|
||||||
|
return putAnswer{Node: node, Module: module, Put: p}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// doPuts is the tool assigner_puts.
|
||||||
|
func (s *putTools) doPuts(args map[string]any) (any, error) {
|
||||||
|
keys, err := s.st.keys(putsState)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the puts cannot be listed: %w", err)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
out := []putAnswer{}
|
||||||
|
for _, key := range keys {
|
||||||
|
node, module, err := delivery.ParsePutKey(key)
|
||||||
|
if err != nil || (arg(args, "node") != "" && node != arg(args, "node")) || (arg(args, "module") != "" && module != arg(args, "module")) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, _, found, err := s.st.get(putsState, key)
|
||||||
|
if err != nil || !found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p, err := delivery.DecodePut(raw)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, putAnswer{Node: node, Module: module, Put: p})
|
||||||
|
}
|
||||||
|
return map[string]any{"puts": out}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func arg(args map[string]any, name string) string {
|
||||||
|
switch v := args[name].(type) {
|
||||||
|
case string:
|
||||||
|
return strings.TrimSpace(v)
|
||||||
|
case bool:
|
||||||
|
return fmt.Sprint(v)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// The tools' names, as the manifest lists them.
|
||||||
|
const (
|
||||||
|
toolPut = "assigner_put"
|
||||||
|
toolRelease = "assigner_release"
|
||||||
|
toolPuts = "assigner_puts"
|
||||||
|
)
|
||||||
|
|
||||||
|
// schema is a tool's arguments, each a string: required ones first named, and switches true or false.
|
||||||
|
func schema(properties map[string]string, required []string, switches ...string) map[string]any {
|
||||||
|
props := map[string]any{}
|
||||||
|
for name, description := range properties {
|
||||||
|
p := map[string]any{"type": "string", "description": description}
|
||||||
|
if slices.Contains(switches, name) {
|
||||||
|
p["enum"] = []string{"true", "false"}
|
||||||
|
}
|
||||||
|
props[name] = p
|
||||||
|
}
|
||||||
|
out := map[string]any{"type": "object", "properties": props}
|
||||||
|
if len(required) > 0 {
|
||||||
|
out["required"] = required
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// tools are the assigner's tools as the runtime serves them (ADR 0546 clause 1).
|
||||||
|
func (s *putTools) tools() []stdio.Tool {
|
||||||
|
return []stdio.Tool{
|
||||||
|
{Name: toolPut, Description: "Put one version of a module on one machine (novox/hq ADR 0546): the build at that fingerprint, " +
|
||||||
|
"the composition and grant the controller wrote for that machine, then the machine's want naming it, held there until " +
|
||||||
|
"released when hold is true. A module whose moves a person takes (the bus, a module of policy record) is asked of the " +
|
||||||
|
"operator first, and the bus has its streams snapshotted after the yes. Answers what was done or what it waits on; the " +
|
||||||
|
"put goes on after the answer, each step said as the event mesh-delivery-assigner.put, and assigner_puts says where it stands.",
|
||||||
|
Input: schema(map[string]string{
|
||||||
|
"node": "the machine, where the module is assigned",
|
||||||
|
"module": "the module",
|
||||||
|
"version": "newest, a fingerprint (src2: and 64 hex digits), or at least its first 12 hex digits",
|
||||||
|
"hold": "\"true\": the want stays at this version until released; \"false\" (the default): the rules move it again once it runs",
|
||||||
|
"why": "why, kept with the put and said in any ask",
|
||||||
|
}, []string{"node", "module", "version", "why"}, "hold"),
|
||||||
|
Run: func(a map[string]any) (any, error) { return s.doPut(a, stdio.CallerOf(a)) }},
|
||||||
|
{Name: toolRelease, Description: "End a put on one machine (novox/hq ADR 0546): its hold is taken away and the assigner's " +
|
||||||
|
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
|
||||||
|
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put"},
|
||||||
|
[]string{"node", "module", "why"}),
|
||||||
|
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
|
||||||
|
{Name: toolPuts, Description: "Every put and where it stands (novox/hq ADR 0546): its version, hold, why, each step with " +
|
||||||
|
"its time and words. Narrowed by machine or module. Read-only.",
|
||||||
|
Input: schema(map[string]string{"node": "one machine", "module": "one module"}, nil),
|
||||||
|
Run: s.doPuts},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runtimeStore is the runtime's states.
|
||||||
|
type runtimeStore struct{}
|
||||||
|
|
||||||
|
func (runtimeStore) get(state, key string) ([]byte, uint64, bool, error) {
|
||||||
|
e, err := stdio.State(state).Get(key)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
return nil, 0, false, err
|
||||||
|
}
|
||||||
|
return e.Value, e.Revision, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runtimeStore) put(state, key string, value []byte) error {
|
||||||
|
_, err := stdio.State(state).Put(key, json.RawMessage(value))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runtimeStore) keys(state string) ([]string, error) { return stdio.State(state).Keys() }
|
||||||
|
|
||||||
|
// serveTools serves the tools to the runtime until it stops, hearing every put to answer assigner_put with its first steps.
|
||||||
|
func serveTools() error {
|
||||||
|
s := newPutTools(runtimeStore{})
|
||||||
|
go func() {
|
||||||
|
<-stdio.Served()
|
||||||
|
for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) {
|
||||||
|
err := stdio.State(putsState).Watch("", func(c stdio.StateChange) error {
|
||||||
|
s.hear(c.Key, c.Value)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Fprintf(os.Stderr, "mesh-delivery-assigner: the puts are not heard yet (%v); asked again in %s\n", err, wait)
|
||||||
|
time.Sleep(wait)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return stdio.Serve("", s.tools())
|
||||||
|
}
|
||||||
|
|
||||||
|
// shortFP is a fingerprint's first twelve hex digits.
|
||||||
|
func shortFP(fp string) string {
|
||||||
|
part, err := delivery.FingerprintKeyPart(fp)
|
||||||
|
if err != nil || len(part) < 12 {
|
||||||
|
return fp
|
||||||
|
}
|
||||||
|
return part[:12]
|
||||||
|
}
|
||||||
@@ -326,6 +326,13 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
|||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// value is what a bucket holds at a key now, nil when nothing.
|
||||||
|
func (w *world) value(bucket, key string) []byte {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
return w.values[bucket][key]
|
||||||
|
}
|
||||||
|
|
||||||
// has says whether a bucket holds a key now.
|
// has says whether a bucket holds a key now.
|
||||||
func (w *world) has(bucket, key string) bool {
|
func (w *world) has(bucket, key string) bool {
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -10,14 +10,17 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
|
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey): keep it held or
|
||||||
// it held or release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19).
|
// release it. Releasing writes the held fingerprint's want again by the rules (ADR 0539 §19). node and words are the held
|
||||||
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
// record's: the machine whose health check failed and the hold's words (issue 1906). asking numbers the writing: 0 the first, a
|
||||||
return delivery.AssignerAsk{Ask: asks.Ask{
|
// larger number an ask written again after a hold lifted by itself, so the router takes it as new.
|
||||||
ID: delivery.AskingID(key, 0),
|
func HoldAsk(key, module, fp, node, words string, asking int, now time.Time) delivery.AssignerAsk {
|
||||||
|
return delivery.AssignerAsk{Asking: asking, Ask: asks.Ask{
|
||||||
|
ID: delivery.AskingID(key, asking),
|
||||||
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||||
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
|
Explanation: fmt.Sprintf("The health check of %s failed on %s at %s (%s), so the assigner holds that version and wants "+
|
||||||
"no node to move to it. Releasing it lets the wants move to it again.", module, short(fp)),
|
"no node to move to it. It lifts the hold by itself once every node that runs %s runs it healthy; releasing it "+
|
||||||
|
"lets the wants move to it now.", module, node, short(fp), words, module),
|
||||||
Options: []asks.Option{
|
Options: []asks.Option{
|
||||||
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
||||||
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
{ID: "release", Label: "Release it", Does: "the version is released and the wants move to it by the rules again", Level: asks.Acknowledge},
|
||||||
@@ -78,6 +81,35 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
|
|||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
|
||||||
|
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
|
||||||
|
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
|
||||||
|
// node and module, and cleared (raised false) when the node has a want for it again.
|
||||||
|
func NeitherCondition(node, module string, h Holding, raised bool) asks.Condition {
|
||||||
|
key := delivery.AssignerModule + ".neither." + node + "." + module
|
||||||
|
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
|
||||||
|
if !raised {
|
||||||
|
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
|
||||||
|
Resolved: fmt.Sprintf("%s on %s has a want again, so its own composition is applied there.", module, node)}
|
||||||
|
}
|
||||||
|
headline := fmt.Sprintf("%s on %s is on no list", module, node)
|
||||||
|
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||||
|
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||||
|
}
|
||||||
|
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||||
|
Summary: headline, Headline: headline,
|
||||||
|
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||||
|
"hold it either, so nothing applies it there. The assigner's rule %d holds its first want back: %s. The assigner writes the want once the "+
|
||||||
|
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
|
||||||
|
func LiftedCondition(module, fp string) asks.Condition {
|
||||||
|
c := HeldCondition(module, fp, "", "", false, false)
|
||||||
|
c.Resolved = LiftedWords(module, fp) + "."
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
// The options of the record ask (ADR 0539 §14) and of the backup ask (§16).
|
||||||
const (
|
const (
|
||||||
OptionYes = "yes"
|
OptionYes = "yes"
|
||||||
|
|||||||
@@ -17,13 +17,25 @@ func TestTheHoldAskIsValid(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
a := HoldAsk(key, "mail", fNew, now)
|
a := HoldAsk(key, "mail", fNew, "ace", "health failed, phase running: connection refused", 0, now)
|
||||||
if err := a.Ask.Check(now); err != nil {
|
if err := a.Ask.Check(now); err != nil {
|
||||||
t.Errorf("the router refuses the ask: %v", err)
|
t.Errorf("the router refuses the ask: %v", err)
|
||||||
}
|
}
|
||||||
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||||
t.Errorf("the ask cannot be written: %v", err)
|
t.Errorf("the ask cannot be written: %v", err)
|
||||||
}
|
}
|
||||||
|
// 1906/H3: the ask names the machine whose check failed and the hold's words.
|
||||||
|
if !strings.Contains(a.Ask.Explanation, "on ace") || !strings.Contains(a.Ask.Explanation, "connection refused") {
|
||||||
|
t.Errorf("the ask does not name the node and the hold's words: %q", a.Ask.Explanation)
|
||||||
|
}
|
||||||
|
// 1906/H2: an ask written again after a lift is another asking, under an id of its own.
|
||||||
|
again := HoldAsk(key, "mail", fNew, "ace", "x", 7, now)
|
||||||
|
if again.Asking != 7 || again.Ask.ID != delivery.AskingID(key, 7) || again.Ask.ID == a.Ask.ID {
|
||||||
|
t.Errorf("the asking after a lift = %d %s", again.Asking, again.Ask.ID)
|
||||||
|
}
|
||||||
|
if _, err := delivery.EncodeAssignerAsk(key, again); err != nil {
|
||||||
|
t.Errorf("the ask asked again cannot be written: %v", err)
|
||||||
|
}
|
||||||
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||||
t.Errorf("the view may not answer it: %v", err)
|
t.Errorf("the view may not answer it: %v", err)
|
||||||
}
|
}
|
||||||
@@ -74,6 +86,17 @@ func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
|
||||||
|
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
|
||||||
|
c := LiftedCondition("mail", fNew)
|
||||||
|
if _, err := asks.EncodeCondition(c); err != nil {
|
||||||
|
t.Fatalf("the condition is refused: %v", err)
|
||||||
|
}
|
||||||
|
if c.State != asks.StateCleared || c.Key != HeldCondition("mail", fNew, "ace", "x", true, false).Key || !strings.Contains(c.Resolved, "lifted its hold by itself") {
|
||||||
|
t.Errorf("condition = %+v", c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
|
||||||
// the view, and name the move and the node.
|
// the view, and name the move and the node.
|
||||||
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
||||||
@@ -123,4 +146,58 @@ func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
|
|||||||
if _, err := asks.EncodeCondition(c); err != nil {
|
if _, err := asks.EncodeCondition(c); err != nil {
|
||||||
t.Errorf("the backup condition is refused: %v", err)
|
t.Errorf("the backup condition is refused: %v", err)
|
||||||
}
|
}
|
||||||
|
// ADR 0546 clauses 7 and 8: a put's ask is valid at the record ask's key; for the bus its yes approves and binds the act,
|
||||||
|
// so the view cannot answer it, and it lasts no longer than an approving ask may.
|
||||||
|
put := delivery.Put{FP: fNew, Why: "the bus's user list", Hold: true}
|
||||||
|
d := delivery.ComposedDigest{Digest: dgst, Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||||
|
for _, bus := range []bool{false, true} {
|
||||||
|
a, err := PutAsk(rkey, "ace", "mail", put, d, bus, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("bus %v: %v", bus, err)
|
||||||
|
}
|
||||||
|
if err := a.Ask.Check(now); err != nil {
|
||||||
|
t.Errorf("bus %v: the router refuses the put's ask: %v", bus, err)
|
||||||
|
}
|
||||||
|
if _, err := delivery.EncodeAssignerAsk(rkey, a); err != nil {
|
||||||
|
t.Errorf("bus %v: the put's ask cannot be written: %v", bus, err)
|
||||||
|
}
|
||||||
|
viewed := delivery.ViewAnswerFits(delivery.ViewAnswer{Asking: a.Ask.ID, Option: OptionYes}, a)
|
||||||
|
yes := a.Ask.Options[0]
|
||||||
|
switch {
|
||||||
|
case !strings.Contains(a.Ask.Explanation, "the bus's user list"):
|
||||||
|
t.Errorf("bus %v: the put's ask does not say why: %s", bus, a.Ask.Explanation)
|
||||||
|
case bus && (yes.Level != asks.Approve || yes.Binds == "" || viewed == nil || a.Ask.Expires.After(now.Add(asks.ApproveLasts))):
|
||||||
|
t.Errorf("bus: the yes is %+v, the view's answer %v, expiring %s", yes, viewed, a.Ask.Expires)
|
||||||
|
case !bus && (yes.Level != asks.Acknowledge || viewed != nil):
|
||||||
|
t.Errorf("a recorded module: the yes is %+v, the view's answer %v", yes, viewed)
|
||||||
|
}
|
||||||
|
later := put
|
||||||
|
later.At = now.Add(time.Minute)
|
||||||
|
b, _ := PutAsk(rkey, "ace", "mail", later, d, bus, now)
|
||||||
|
if b.Ask.ID == a.Ask.ID || b.Asking != PutAsking(later) {
|
||||||
|
t.Errorf("bus %v: a later put's ask is asking %d under id %q, the earlier's %q", bus, b.Asking, b.Ask.ID, a.Ask.ID)
|
||||||
|
}
|
||||||
|
if err := yes.Performs(PutAct("ace", "mail", fNew, dgst)); err != nil {
|
||||||
|
t.Errorf("bus %v: the yes does not perform the act it bound: %v", bus, err)
|
||||||
|
}
|
||||||
|
if err := yes.Performs(PutAct("ace", "mail", fNew, dig(fOld))); bus && err == nil {
|
||||||
|
t.Errorf("bus: the yes performs another composition's act")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC21: the condition of a module on neither list is urgent, names the module, the node and the rule holding its want back,
|
||||||
|
// under the assigner's own name keyed by node and module, and is cleared under the same key.
|
||||||
|
func TestTheNeitherConditionNamesModuleNodeAndRule(t *testing.T) {
|
||||||
|
c := NeitherCondition("g14", "i3status-rust", Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}, true)
|
||||||
|
if _, err := asks.EncodeCondition(c); err != nil {
|
||||||
|
t.Fatalf("the condition is refused: %v", err)
|
||||||
|
}
|
||||||
|
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Subject.ID != "i3status-rust" || c.Subject.Machine != "g14" ||
|
||||||
|
!strings.Contains(c.Explanation, "x11-display") || !strings.Contains(c.Explanation, "rule 5") || c.Key != "mesh-delivery-assigner.neither.g14.i3status-rust" {
|
||||||
|
t.Errorf("condition = %+v", c)
|
||||||
|
}
|
||||||
|
if got := NeitherCondition("g14", "i3status-rust", Holding{}, false); got.State != asks.StateCleared || got.Key != c.Key {
|
||||||
|
t.Errorf("cleared = %+v", got)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,252 @@
|
|||||||
|
package assign
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A put (novox/hq ADR 0546): one version of a module put on one node by the assigner's tool assigner_put. The tool writes the put
|
||||||
|
// key; each pass takes it one step further from what the world holds (StepPut), and the rules pass over the node's want for
|
||||||
|
// the module from its asking until it ends (Module.Puts).
|
||||||
|
|
||||||
|
// ComposeWithin is how long a put waits for the composition and grant it needs (a take-in of a build just done writes them)
|
||||||
|
// before it is refused.
|
||||||
|
const ComposeWithin = 10 * time.Minute
|
||||||
|
|
||||||
|
// PutWorld is what one step of a put reads.
|
||||||
|
type PutWorld struct {
|
||||||
|
Node, Module string
|
||||||
|
Put delivery.Put
|
||||||
|
// Assigned says the module stands assigned to the node.
|
||||||
|
Assigned bool
|
||||||
|
// Bus says the module provides the mesh's bus (ADR 0539 §15); Recorded that it is of policy record (§14).
|
||||||
|
Bus, Recorded bool
|
||||||
|
// Build is the state of the builder's build at the put's fingerprint, "" when no key is written; BuildWords its error.
|
||||||
|
Build, BuildWords string
|
||||||
|
// Newest is the module's newest fingerprint at the trunk, which the builder builds by itself.
|
||||||
|
Newest string
|
||||||
|
// Composed is the composition of the module for the node at the fingerprint, as its writer holds it; nil when there is
|
||||||
|
// none. Granted says the grant at the fingerprint names the node's accounts. Both are read, never asked for (clause 4).
|
||||||
|
Composed *delivery.ComposedDigest
|
||||||
|
Granted bool
|
||||||
|
// Answer is the option the operator chose on this put's own ask (its asking, PutAsking), "" when none; AskExpired says
|
||||||
|
// that ask expired unanswered.
|
||||||
|
Answer string
|
||||||
|
AskExpired bool
|
||||||
|
// Snapshot says where the bus's streams were snapshotted for this put, once they were; Snapshotting that it was
|
||||||
|
// asked; SnapshotFailed why it was not taken.
|
||||||
|
Snapshot string
|
||||||
|
Snapshotting bool
|
||||||
|
SnapshotFailed string
|
||||||
|
// Have is what the node says it runs of the module; nil when it says nothing.
|
||||||
|
Have *delivery.Have
|
||||||
|
Now time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutNext is what a step decides: the put as it stands after it, and the acts the pass does for it.
|
||||||
|
type PutNext struct {
|
||||||
|
Put delivery.Put
|
||||||
|
// Ask writes the put's ask about the move (PutAsk).
|
||||||
|
Ask bool
|
||||||
|
// Snapshot has the bus's node back the bus up now, its dump the streams' snapshot (ADR 0235).
|
||||||
|
Snapshot bool
|
||||||
|
// Warranted records, once, in the controller's hand-act log that the bus's want was written on the operator's warrant.
|
||||||
|
Warranted bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// StepPut takes a put one step further from what the world holds (ADR 0546 clauses 3 to 8).
|
||||||
|
func StepPut(w PutWorld) PutNext {
|
||||||
|
p := w.Put
|
||||||
|
if delivery.PutEnded(p.Step) {
|
||||||
|
return PutNext{Put: p}
|
||||||
|
}
|
||||||
|
say := func(step, format string, args ...any) PutNext {
|
||||||
|
return PutNext{Put: p.Said(w.Now, step, fmt.Sprintf(format, args...))}
|
||||||
|
}
|
||||||
|
fp := short(p.FP)
|
||||||
|
if !w.Assigned {
|
||||||
|
return say(delivery.PutRefused, "%s is not assigned to %s: a put moves a module where it stands assigned (mesh-controller.assign first)", w.Module, w.Node)
|
||||||
|
}
|
||||||
|
switch w.Build {
|
||||||
|
case delivery.BuildDone:
|
||||||
|
case delivery.BuildFailed:
|
||||||
|
return say(delivery.PutRefused, "the builder's build of %s at %s failed: %s", w.Module, fp, w.BuildWords)
|
||||||
|
case "":
|
||||||
|
if p.FP != w.Newest {
|
||||||
|
// The builder builds the trunk's newest fingerprint and the candidates', and takes no ask for another yet (ADR 0546
|
||||||
|
// clause 3): an older fingerprint it never built, or whose key it removed, cannot be put.
|
||||||
|
return say(delivery.PutRefused, "the builder holds no build of %s at %s, and builds by itself only the trunk's newest (%s): put that, or a version it built", w.Module, fp, short(w.Newest))
|
||||||
|
}
|
||||||
|
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, the trunk's newest, which it builds by itself", w.Module, fp)
|
||||||
|
default:
|
||||||
|
return say(delivery.PutBuild, "waits for the builder's build of %s at %s, which is %s", w.Module, fp, w.Build)
|
||||||
|
}
|
||||||
|
if w.Composed == nil || !w.Granted {
|
||||||
|
missing := fmt.Sprintf("no composition of %s for %s at %s is written", w.Module, w.Node, fp)
|
||||||
|
if w.Composed != nil {
|
||||||
|
missing = fmt.Sprintf("the grant of %s at %s does not name %s's accounts", w.Module, fp, w.Node)
|
||||||
|
}
|
||||||
|
since := w.Now
|
||||||
|
if p.Step == delivery.PutCompose && len(p.Steps) > 0 {
|
||||||
|
since = p.Steps[len(p.Steps)-1].At
|
||||||
|
}
|
||||||
|
if w.Now.Sub(since) > ComposeWithin {
|
||||||
|
return say(delivery.PutRefused, "%s after %s: only a composition written by the take-in of its build (or one of the five kept) is put; "+
|
||||||
|
"for another the hand path is mesh-hand-compose (ADR 0539 §10)", missing, ComposeWithin)
|
||||||
|
}
|
||||||
|
return say(delivery.PutCompose, "waits up to %s: %s yet", ComposeWithin, missing)
|
||||||
|
}
|
||||||
|
d := *w.Composed
|
||||||
|
if p.Step == delivery.PutWanted || p.Step == delivery.PutHolding {
|
||||||
|
return PutNext{Put: applied(w, p)}
|
||||||
|
}
|
||||||
|
p.Composed = d.Digest
|
||||||
|
if needsPerson(w, d) {
|
||||||
|
switch w.Answer {
|
||||||
|
case OptionNo:
|
||||||
|
return say(delivery.PutRefused, "the operator answered no to putting %s at %s on %s", w.Module, fp, w.Node)
|
||||||
|
case OptionYes:
|
||||||
|
default:
|
||||||
|
if w.AskExpired {
|
||||||
|
return say(delivery.PutRefused, "the operator's ask about putting %s at %s on %s expired unanswered: the node keeps what it runs", w.Module, fp, w.Node)
|
||||||
|
}
|
||||||
|
next := say(delivery.PutPerson, "waits for the operator's yes to putting %s at %s on %s (composition %s, judged %s)", w.Module, fp, w.Node, short(d.Digest), d.Judgement)
|
||||||
|
if p.Ask == "" {
|
||||||
|
next.Ask = true
|
||||||
|
next.Put.Ask, _ = delivery.RecordAskKey(w.Node, w.Module, d.Digest)
|
||||||
|
}
|
||||||
|
return next
|
||||||
|
}
|
||||||
|
if w.Bus {
|
||||||
|
switch {
|
||||||
|
case w.Snapshot != "":
|
||||||
|
p.Snapshot = w.Snapshot
|
||||||
|
case w.SnapshotFailed != "":
|
||||||
|
return say(delivery.PutRefused, "the bus's streams were not snapshotted, so the bus is not put: %s", w.SnapshotFailed)
|
||||||
|
case w.Snapshotting:
|
||||||
|
return say(delivery.PutSnapshot, "the operator said yes; waits for %s's backup holder to snapshot the bus's streams", w.Node)
|
||||||
|
default:
|
||||||
|
next := say(delivery.PutSnapshot, "the operator said yes; asked %s's backup holder to snapshot the bus's streams", w.Node)
|
||||||
|
next.Snapshot = true
|
||||||
|
return next
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
words := fmt.Sprintf("the want of %s on %s names %s (composition %s)", w.Module, w.Node, fp, short(d.Digest))
|
||||||
|
if p.Hold {
|
||||||
|
words += ", held until released"
|
||||||
|
}
|
||||||
|
if p.Snapshot != "" {
|
||||||
|
words += "; the streams were snapshotted: " + p.Snapshot
|
||||||
|
}
|
||||||
|
next := PutNext{Put: applied(w, p.Said(w.Now, delivery.PutWanted, words))}
|
||||||
|
next.Warranted = w.Bus && p.Ask != ""
|
||||||
|
return next
|
||||||
|
}
|
||||||
|
|
||||||
|
// applied is a wanted put once the node's have says how it went: holding or done when it runs the composition healthy.
|
||||||
|
func applied(w PutWorld, p delivery.Put) delivery.Put {
|
||||||
|
h := w.Have
|
||||||
|
if h == nil || h.FP != p.FP || h.Composed != p.Composed {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK && p.Hold:
|
||||||
|
return p.Said(w.Now, delivery.PutHolding, fmt.Sprintf("%s runs %s at %s healthy; held there until released", w.Node, w.Module, short(p.FP)))
|
||||||
|
case h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK:
|
||||||
|
return p.Said(w.Now, delivery.PutDone, fmt.Sprintf("%s runs %s at %s healthy; the rules write its want again", w.Node, w.Module, short(p.FP)))
|
||||||
|
case failed(*h):
|
||||||
|
return p.Said(w.Now, p.Step, fmt.Sprintf("%s runs %s at %s and its health check failed: %s", w.Node, w.Module, short(p.FP), h.Detail))
|
||||||
|
}
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// needsPerson says whether the put's move waits for the operator: any move of the bus but a revision at the fingerprint the
|
||||||
|
// node runs that changes only its user list (ADR 0539 §7, §15), and a move of a module of policy record judged to recreate a
|
||||||
|
// container, move data, or that cannot be judged (§14).
|
||||||
|
func needsPerson(w PutWorld, d delivery.ComposedDigest) bool {
|
||||||
|
runs := w.Have != nil && w.Have.FP == w.Put.FP
|
||||||
|
if runs && w.Have.Composed == d.Digest {
|
||||||
|
return false // the node runs this composition already: nothing moves
|
||||||
|
}
|
||||||
|
if w.Bus {
|
||||||
|
return !(runs && d.UserListOnly)
|
||||||
|
}
|
||||||
|
if !w.Recorded {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return d.Judgement == delivery.JudgedRecreates || d.Judgement == delivery.JudgedMovesData || d.Judgement == delivery.JudgedCannot
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutWant is the want a put writes once it is wanted: the fingerprint and composition, marked put, and held when it holds.
|
||||||
|
func PutWant(p delivery.Put) delivery.Want {
|
||||||
|
return delivery.Want{FP: p.FP, Composed: p.Composed, Put: true, Hold: p.Hold, Words: "put: " + p.Why}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutOwns says what a put does to the node's want: owns says the rules pass over it, want the want it writes (nil: what
|
||||||
|
// stands stays). A put owns the want from its asking until it ends; a held put ends only when released.
|
||||||
|
func PutOwns(p delivery.Put) (want *delivery.Want, owns bool) {
|
||||||
|
switch p.Step {
|
||||||
|
case delivery.PutWanted, delivery.PutHolding:
|
||||||
|
w := PutWant(p)
|
||||||
|
return &w, true
|
||||||
|
case delivery.PutDone, delivery.PutReleased, delivery.PutRefused:
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return nil, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutAsking is a put's own asking number: when it was asked, in milliseconds. Each put asked under the same key is another
|
||||||
|
// asking, so the ask is written again under an id of its own and an answer to an earlier put, whose warrant names that put's
|
||||||
|
// asking, never counts for a later one (ADR 0546 clause 8: one approval binds one act).
|
||||||
|
func PutAsking(p delivery.Put) int {
|
||||||
|
if n := p.At.UnixMilli(); n > 0 {
|
||||||
|
return int(n)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutAct is the act a bus put's yes binds and that is performed when its want is written.
|
||||||
|
func PutAct(node, module, fp, composed string) asks.Act {
|
||||||
|
return asks.Act{"put": "want", "node": node, "module": module, "fp": fp, "composed": composed}
|
||||||
|
}
|
||||||
|
|
||||||
|
// PutAsk is the ask about a put's move (ADR 0546 clauses 7 and 8), under the record ask's key of the composition so the
|
||||||
|
// teller relays it and its answer as any record ask's. For the bus its yes is of level approve and binds the act, so only a
|
||||||
|
// channel that proves who answered takes it (ADR 0259), never the view.
|
||||||
|
func PutAsk(key, node, module string, p delivery.Put, d delivery.ComposedDigest, bus bool, now time.Time) (delivery.AssignerAsk, error) {
|
||||||
|
yes := asks.Option{ID: OptionYes, Label: "Yes, put it", Does: "the node is told to apply " + module + " at " + short(p.FP), Level: asks.Acknowledge}
|
||||||
|
what := map[delivery.Judgement]string{
|
||||||
|
delivery.JudgedRecreates: "recreates a container",
|
||||||
|
delivery.JudgedMovesData: "moves the module's data",
|
||||||
|
delivery.JudgedCannot: "cannot be judged: " + d.Words,
|
||||||
|
delivery.JudgedNeither: "recreates no container and moves no data",
|
||||||
|
delivery.JudgedUnchanged: "changes nothing the node runs",
|
||||||
|
}[d.Judgement]
|
||||||
|
explanation := fmt.Sprintf("Asked to put %s at version %s on %s: %s. The move %s.", module, short(p.FP), node, p.Why, what)
|
||||||
|
expires := now.Add(7 * 24 * time.Hour)
|
||||||
|
if bus {
|
||||||
|
binds, err := asks.ActDigest(PutAct(node, module, p.FP, d.Digest))
|
||||||
|
if err != nil {
|
||||||
|
return delivery.AssignerAsk{}, err
|
||||||
|
}
|
||||||
|
yes.Level, yes.Binds = asks.Approve, binds
|
||||||
|
yes.Does = "the bus's streams are snapshotted, then " + node + " is told to apply the bus at " + short(p.FP)
|
||||||
|
explanation += " It is the mesh's bus: every module reconnects while it restarts, and its streams are snapshotted first."
|
||||||
|
expires = now.Add(asks.ApproveLasts)
|
||||||
|
}
|
||||||
|
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||||
|
ID: delivery.AskingID(key, PutAsking(p)),
|
||||||
|
Headline: fmt.Sprintf("Put %s on %s?", module, node),
|
||||||
|
Explanation: explanation,
|
||||||
|
Options: []asks.Option{yes,
|
||||||
|
{ID: OptionNo, Label: "No", Does: "the node keeps what it runs, and the put is refused", Level: asks.Acknowledge}},
|
||||||
|
Who: asks.Operator,
|
||||||
|
Expires: expires,
|
||||||
|
OnExpiry: "the node keeps what it runs",
|
||||||
|
About: key,
|
||||||
|
}, Asking: PutAsking(p)}, nil
|
||||||
|
}
|
||||||
@@ -4,6 +4,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"slices"
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -13,13 +15,20 @@ import (
|
|||||||
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
|
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
|
||||||
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
|
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
|
||||||
type Module struct {
|
type Module struct {
|
||||||
Name string
|
Name string
|
||||||
FP string
|
FP string
|
||||||
Built bool
|
Built bool
|
||||||
Held bool
|
Held bool
|
||||||
Granted bool
|
Granted bool
|
||||||
Manifest catalogue.Manifest
|
// GrantedAt says whether the controller issued the module's credential for another fingerprint than FP: the first want of a node
|
||||||
Nodes []string
|
// that runs an older one names it (issue 1945). Nil: none.
|
||||||
|
GrantedAt func(fp string) bool
|
||||||
|
// GrantedOn says whether the controller's grant of FP names the module's accounts on a node, when the grant names any:
|
||||||
|
// a grant written for the module's other nodes says nothing of a node whose account is not issued and sealed (issue
|
||||||
|
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
|
||||||
|
GrantedOn func(node string) bool
|
||||||
|
Manifest catalogue.Manifest
|
||||||
|
Nodes []string
|
||||||
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
|
||||||
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
|
||||||
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
Composed func(node, fp string) (delivery.ComposedDigest, bool)
|
||||||
@@ -31,6 +40,12 @@ type Module struct {
|
|||||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||||
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
// as any module is, losing the writes made since (ADR 0539 §19). Nil: none did.
|
||||||
Restored func(node string) bool
|
Restored func(node string) bool
|
||||||
|
// Kept says the operator answered "keep" to the hold ask of FP: a hold kept so is not lifted by itself.
|
||||||
|
Kept bool
|
||||||
|
// Now is when this pass reads the haves: the held record names it as when the health check was seen to fail. Zero: not said.
|
||||||
|
Now time.Time
|
||||||
|
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
||||||
|
Puts map[string]delivery.Put
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
|
||||||
@@ -69,27 +84,40 @@ func Canary(nodes []string, online map[string]bool) string {
|
|||||||
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
||||||
// limit (ADR 0528 clause 3).
|
// limit (ADR 0528 clause 3).
|
||||||
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||||
|
return holds(m, node, n, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
|
||||||
|
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
|
||||||
|
// composition.
|
||||||
|
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||||
switch {
|
switch {
|
||||||
case !m.Built:
|
case !m.Built:
|
||||||
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
|
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
|
||||||
case m.Held:
|
case m.Held:
|
||||||
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
|
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
|
||||||
}
|
}
|
||||||
canary := Canary(m.Nodes, n.Online)
|
if ordering {
|
||||||
switch {
|
canary := Canary(m.Nodes, n.Online)
|
||||||
case canary == "":
|
switch {
|
||||||
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
case canary == "":
|
||||||
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
||||||
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||||
}
|
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||||
for _, provision := range m.Manifest.Requires {
|
}
|
||||||
if ok, why := providedHealthy(provision, node, n); !ok {
|
for _, provision := range m.Manifest.Requires {
|
||||||
return false, delivery.RuleProviders, why
|
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||||
|
return false, delivery.RuleProviders, why
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !m.Granted {
|
if !m.Granted {
|
||||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||||
}
|
}
|
||||||
|
if m.GrantedOn != nil && !m.GrantedOn(node) {
|
||||||
|
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
|
||||||
|
short(m.FP))
|
||||||
|
}
|
||||||
if _, ok := m.composition(node, m.FP); !ok {
|
if _, ok := m.composition(node, m.FP); !ok {
|
||||||
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
|
||||||
}
|
}
|
||||||
@@ -138,6 +166,9 @@ type Outcome struct {
|
|||||||
Wants map[string]delivery.Want
|
Wants map[string]delivery.Want
|
||||||
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
||||||
Held *delivery.Held
|
Held *delivery.Held
|
||||||
|
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
|
||||||
|
// record and its ask are taken away, and the wants move by the rules again in this pass.
|
||||||
|
Lifted bool
|
||||||
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
|
||||||
Record []RecordNeed
|
Record []RecordNeed
|
||||||
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
|
||||||
@@ -145,6 +176,11 @@ type Outcome struct {
|
|||||||
Restore []string
|
Restore []string
|
||||||
// Rules is, by node, the rule holding the node back from m.FP and why.
|
// Rules is, by node, the rule holding the node back from m.FP and why.
|
||||||
Rules map[string]Holding
|
Rules map[string]Holding
|
||||||
|
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
|
||||||
|
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
|
||||||
|
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
|
||||||
|
// own composition and is never here.
|
||||||
|
Neither map[string]Holding
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
|
// RecordNeed is a move of a module of policy record on a node that waits for the operator's yes: the composition's digest, the
|
||||||
@@ -168,12 +204,15 @@ type Holding struct {
|
|||||||
// module holding data is not set back but asked about (Restore). A
|
// module holding data is not set back but asked about (Restore). A
|
||||||
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
||||||
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
||||||
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
|
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}, Neither: map[string]Holding{}}
|
||||||
held := m.Held
|
held := m.Held
|
||||||
if !held && !m.Released && m.FP != "" {
|
if held && !m.Kept && m.FP != "" && allRunHealthy(m, n) {
|
||||||
|
out.Lifted, held = true, false
|
||||||
|
}
|
||||||
|
if !held && !out.Lifted && !m.Released && m.FP != "" {
|
||||||
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||||
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
|
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
|
||||||
out.Held = &delivery.Held{Node: node, Words: h.Detail}
|
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
|
||||||
held = true
|
held = true
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
@@ -187,6 +226,21 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
|||||||
}
|
}
|
||||||
before, had := wants[node]
|
before, had := wants[node]
|
||||||
var next delivery.Want
|
var next delivery.Want
|
||||||
|
if p, put := m.Puts[node]; put {
|
||||||
|
if want, owns := PutOwns(p); owns {
|
||||||
|
if want != nil {
|
||||||
|
w := *want
|
||||||
|
// A held bus still takes a revision of its composition that changes only its user list (ADR 0539 §7).
|
||||||
|
if ul, ok := m.userListRevision(node, before, had, n); ok && ul.FP == w.FP {
|
||||||
|
w.Composed = ul.Composed
|
||||||
|
}
|
||||||
|
if !had || !delivery.SameWant(normal(before), normal(w)) {
|
||||||
|
out.Wants[node] = w
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
// A revision of the bus module's composition that differs only in its user list is written at once, at the fingerprint the
|
||||||
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
// node runs, outside the rules, the record hold and the bar on a new fingerprint: a new module's credential must reach the
|
||||||
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
// bus before the module can connect (ADR 0539 §7). The controller marks it; whatever it judges of the file's mount stands.
|
||||||
@@ -196,15 +250,49 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
switch ok, rule, words := Holds(mm, node, n); {
|
ok, rule, words := Holds(mm, node, n)
|
||||||
|
target := m.FP
|
||||||
|
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
|
||||||
|
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
|
||||||
|
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
|
||||||
|
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
|
||||||
|
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
|
||||||
|
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
|
||||||
|
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
|
||||||
|
// order the move from it as from any want.
|
||||||
|
switch _, composed := m.composition(node, have.FP); {
|
||||||
|
case !ran:
|
||||||
|
case have.FP == m.FP:
|
||||||
|
ok, rule, words = holds(mm, node, n, false)
|
||||||
|
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
|
||||||
|
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
|
||||||
|
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
|
||||||
|
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
|
||||||
|
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
|
||||||
|
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
|
||||||
|
ok, target = true, have.FP
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
case ok:
|
case ok:
|
||||||
d, _ := m.composition(node, m.FP)
|
d, _ := m.composition(node, target)
|
||||||
next = delivery.Want{FP: m.FP, Composed: d.Digest}
|
next = delivery.Want{FP: target, Composed: d.Digest}
|
||||||
if had && !before.Removed && before.FP == m.FP && before.Composed == d.Digest {
|
at := m
|
||||||
|
if target != m.FP {
|
||||||
|
// the first want at the older fingerprint the node runs: held back from F by the rule, and judged as a want at it
|
||||||
|
next.Newest, next.Rule, next.Words = m.FP, rule, words
|
||||||
|
at.FP = target
|
||||||
|
}
|
||||||
|
if had && !before.Removed && !before.Put && before.FP == target && before.Composed == d.Digest {
|
||||||
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
next.Backup, next.Words = before.Backup, before.Words // the answer to a failed backup stays while its target stays (ADR 0539 §16)
|
||||||
}
|
}
|
||||||
running, ran := n.Haves[node][m.Name]
|
running, ran := n.Haves[node][m.Name]
|
||||||
if need, wait := m.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
|
// A removed want is no want: a first want after it goes through the record ask as a first want does.
|
||||||
|
asked, hadLive := before, had
|
||||||
|
if before.Removed {
|
||||||
|
asked, hadLive = delivery.Want{}, false
|
||||||
|
}
|
||||||
|
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
||||||
if need != nil {
|
if need != nil {
|
||||||
out.Record = append(out.Record, *need)
|
out.Record = append(out.Record, *need)
|
||||||
}
|
}
|
||||||
@@ -233,11 +321,33 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
|||||||
default:
|
default:
|
||||||
out.Rules[node] = Holding{Rule: rule, Words: words}
|
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||||
if !had || before.Removed {
|
if !had || before.Removed {
|
||||||
|
// The node runs the module, or its have names it, and gets no want for it: the module is on neither list (issue 1945), and
|
||||||
|
// the pass says so. A new assignment, with no have, waits for the rules and is on no list yet by design. A held module (rule 2,
|
||||||
|
// held before or by this pass) is on the held list instead, whose hold is said and answered on its own: it is never on neither
|
||||||
|
// list (issue 1945: systemd-resolved, held on shanks by the operator, was said as urgent every hour).
|
||||||
|
if h, there := n.Haves[node][m.Name]; there && h.Phase != delivery.PhaseRemoved && !held {
|
||||||
|
out.Neither[node] = Holding{Rule: rule, Words: words}
|
||||||
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
|
next = delivery.Want{FP: before.FP, Composed: before.Composed, Newest: m.FP, Rule: rule, Words: words}
|
||||||
if before.FP == m.FP {
|
if before.FP == m.FP {
|
||||||
|
// Held at F by the rules: a new revision of F's composition waits for them, as a new fingerprint does (ADR 0539 §5).
|
||||||
next = delivery.Want{FP: m.FP, Composed: before.Composed}
|
next = delivery.Want{FP: m.FP, Composed: before.Composed}
|
||||||
|
} else if d, there := m.composition(node, before.FP); there && d.Digest != "" && d.Digest != before.Composed {
|
||||||
|
// Held back at an earlier fingerprint whose kept composition the controller composed again with today's config
|
||||||
|
// (novox/hq ADR 0545): the want follows its new revision, so the node does not run that build with the config of
|
||||||
|
// the moment it was made (on g14 a capacity of 0 after a rollback). Asked as any move of a record module is.
|
||||||
|
at := m
|
||||||
|
at.FP = before.FP
|
||||||
|
running, ran := n.Haves[node][m.Name]
|
||||||
|
need, wait := at.mustAsk(node, before, true, ran, running.FP, running.Composed == "", d)
|
||||||
|
if need != nil {
|
||||||
|
out.Record = append(out.Record, *need)
|
||||||
|
}
|
||||||
|
if !wait {
|
||||||
|
next.Composed = d.Digest
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
||||||
@@ -252,6 +362,58 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// failed says whether a have's health check failed once the node is done moving: phase running or failed (never removed, held, or
|
||||||
|
// moving). A failed health while it
|
||||||
|
// backs up, applies or checks is a step of the move, not its outcome (on novox at 15:27:36 on 2026-10-25, "health failed, phase
|
||||||
|
// applying" held the controller's canary two seconds before the new build was healthy). Every healthy read (healthy, providedHealthy,
|
||||||
|
// the world's lastHealthy) already asks for phase running.
|
||||||
|
func failed(h delivery.Have) bool {
|
||||||
|
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// runs says whether a have is of a module the node runs: running, whatever its health.
|
||||||
|
func runs(h delivery.Have) bool {
|
||||||
|
return h.Phase == delivery.PhaseRunning
|
||||||
|
}
|
||||||
|
|
||||||
|
// allRunHealthy says whether every node of m that runs the module (its have's phase is running, or failed) runs m.FP healthy, and
|
||||||
|
// at least one does: a hold whose fingerprint runs healthy wherever the module runs holds nothing back any more (issue 1906: seven
|
||||||
|
// modules stayed held on every machine after one transient failed check). A node set back to another fingerprint does not run
|
||||||
|
// m.FP, and a failed one does not run it healthy, so the hold stays; a have that is removed, held for its backup or still moving
|
||||||
|
// (backing up, applying, checking) is not counted, or it would keep the hold up for ever.
|
||||||
|
func allRunHealthy(m Module, n Nodes) bool {
|
||||||
|
ran := 0
|
||||||
|
for _, node := range m.Nodes {
|
||||||
|
h, ok := n.Haves[node][m.Name]
|
||||||
|
if !ok || (h.Phase != delivery.PhaseRunning && h.Phase != delivery.PhaseFailed) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !healthy(h, m.FP) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
ran++
|
||||||
|
}
|
||||||
|
return ran > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeldWords are the words a hold keeps of its cause (issue 1906): the health and phase the node reported, when the assigner saw
|
||||||
|
// it, and the node's own words. The node is the held record's own field.
|
||||||
|
func HeldWords(h delivery.Have, at time.Time) string {
|
||||||
|
words := fmt.Sprintf("health %s, phase %s", h.Health, h.Phase)
|
||||||
|
if !at.IsZero() {
|
||||||
|
words += ", seen at " + at.UTC().Format(time.RFC3339)
|
||||||
|
}
|
||||||
|
if d := strings.TrimSpace(h.Detail); d != "" {
|
||||||
|
return words + ": " + d
|
||||||
|
}
|
||||||
|
return words + ": the node gave no words"
|
||||||
|
}
|
||||||
|
|
||||||
|
// LiftedWords say why a hold was lifted by itself.
|
||||||
|
func LiftedWords(module, fp string) string {
|
||||||
|
return fmt.Sprintf("every node that runs %s runs %s healthy, so the assigner lifted its hold by itself", module, short(fp))
|
||||||
|
}
|
||||||
|
|
||||||
// normal is a want without its version, as a decoded and a composed one are compared.
|
// normal is a want without its version, as a decoded and a composed one are compared.
|
||||||
func normal(w delivery.Want) delivery.Want {
|
func normal(w delivery.Want) delivery.Want {
|
||||||
w.V = 0
|
w.V = 0
|
||||||
@@ -329,3 +491,9 @@ func (m Module) userListRevision(node string, before delivery.Want, had bool, n
|
|||||||
}
|
}
|
||||||
return w, true
|
return w, true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GrantNamesNode says whether a grant of module names its accounts on node (mesh-sdk's ControllerGrant.NamesNode, which a
|
||||||
|
// put reads by too).
|
||||||
|
func GrantNamesNode(g delivery.ControllerGrant, node, module string) bool {
|
||||||
|
return g.NamesNode(node, module)
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -85,6 +86,33 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
|||||||
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
|
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
|
||||||
|
// (issue 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus).
|
||||||
|
m, n := ready()
|
||||||
|
grant := delivery.ControllerGrant{Accounts: []string{"ace.mail", "ace.mail.send"}}
|
||||||
|
m.GrantedOn = func(node string) bool { return GrantNamesNode(grant, node, "mail") }
|
||||||
|
out := Wants(m, n, wantedOld(), nil)
|
||||||
|
if w := out.Wants["ace"]; w.FP != fNew {
|
||||||
|
t.Errorf("ace, whose account the grant names, wants %+v", w)
|
||||||
|
}
|
||||||
|
if got := out.Rules["g14"]; got.Rule != delivery.RuleGrant || !strings.Contains(got.Words, "on g14") {
|
||||||
|
t.Errorf("g14, whose account the grant does not name, is held by %+v", got)
|
||||||
|
}
|
||||||
|
for _, c := range []struct {
|
||||||
|
accounts []string
|
||||||
|
node, module string
|
||||||
|
named bool
|
||||||
|
}{
|
||||||
|
{nil, "g14", "mail", true},
|
||||||
|
{[]string{"g14.mail.send"}, "g14", "mail", true},
|
||||||
|
{[]string{"g14.mailer"}, "g14", "mail", false},
|
||||||
|
{[]string{"ace.mail"}, "g14", "mail", false},
|
||||||
|
} {
|
||||||
|
if got := GrantNamesNode(delivery.ControllerGrant{Accounts: c.accounts}, c.node, c.module); got != c.named {
|
||||||
|
t.Errorf("%v names %s of %s: %v, want %v", c.accounts, c.module, c.node, got, c.named)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first.
|
// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first.
|
||||||
@@ -110,8 +138,10 @@ func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
|
|||||||
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||||
m, n := ready()
|
m, n := ready()
|
||||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
|
||||||
|
m.Now = time.Date(2026, 10, 25, 1, 0, 0, 0, time.UTC)
|
||||||
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fOld, Composed: dig(fOld)}}, map[string]string{"ace": fOld})
|
||||||
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
|
// 1906/H3: the held record keeps its cause in words: which machine (its node), which health, when, and the node's words.
|
||||||
|
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "health failed, phase running, seen at 2026-10-25T01:00:00Z: connection refused" {
|
||||||
t.Fatalf("held = %+v", out.Held)
|
t.Fatalf("held = %+v", out.Held)
|
||||||
}
|
}
|
||||||
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
if w := out.Wants["ace"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
||||||
@@ -218,6 +248,25 @@ func TestARevisionAtTheSameFingerprintIsAWantByTheSameRules(t *testing.T) {
|
|||||||
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
|
if w, ok := out.Wants["ace"]; ok && w.Composed == revised {
|
||||||
t.Errorf("a revision was written while a rule holds: %+v", w)
|
t.Errorf("a revision was written while a rule holds: %+v", w)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A want held back at an earlier fingerprint follows the new revision of that fingerprint's kept composition, which the
|
||||||
|
// controller composed again with today's config (novox/hq ADR 0545: on g14 a capacity of 0 after a rollback), while
|
||||||
|
// the rules still hold F; for a record module whose revision recreates a container, only after the ask.
|
||||||
|
out = Wants(m, n, wantedOld(), nil)
|
||||||
|
if w := out.Wants["g14"]; w.FP != fOld || w.Composed != revised || w.Newest != fNew {
|
||||||
|
t.Errorf("g14, held back at its earlier fingerprint, wants %+v; want it at that fingerprint naming the revised digest", w)
|
||||||
|
}
|
||||||
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||||
|
return delivery.ComposedDigest{Digest: revised, Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord,
|
||||||
|
Against: dig(fOld), Words: "recreates"}, true
|
||||||
|
}
|
||||||
|
out = Wants(m, n, wantedOld(), nil)
|
||||||
|
if w := out.Wants["g14"]; w.Composed == revised {
|
||||||
|
t.Errorf("a recorded module's revision that recreates a container was written with no ask: %+v", w)
|
||||||
|
}
|
||||||
|
if !slices.ContainsFunc(out.Record, func(r RecordNeed) bool { return r.Node == "g14" && r.Digest == revised }) {
|
||||||
|
t.Errorf("the revision was not asked: %+v", out.Record)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
|
// 1906/D2: a set-back names the composition of the fingerprint it sets back to, and when that is gone the want stays and says so.
|
||||||
@@ -301,6 +350,7 @@ func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T)
|
|||||||
}
|
}
|
||||||
m, n, _ = movingTo(delivery.JudgedRecreates)
|
m, n, _ = movingTo(delivery.JudgedRecreates)
|
||||||
recording(&m)
|
recording(&m)
|
||||||
|
m.Nodes = []string{"ace"} // g14 runs it with no want, so its first want is asked (1945/AC19): this case is ace's alone
|
||||||
delete(n.Haves["ace"], "mail")
|
delete(n.Haves["ace"], "mail")
|
||||||
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
if out := Wants(m, n, nil, nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 {
|
||||||
t.Errorf("a node that never ran it: %+v", out)
|
t.Errorf("a node that never ran it: %+v", out)
|
||||||
@@ -334,6 +384,137 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
|||||||
if len(out.Record) != 0 {
|
if len(out.Record) != 0 {
|
||||||
t.Errorf("a user-list revision was asked: %+v", out.Record)
|
t.Errorf("a user-list revision was asked: %+v", out.Record)
|
||||||
}
|
}
|
||||||
|
// ADR 0546: a put owns the node's want from its asking until it ends. While it waits the rules write nothing there; once
|
||||||
|
// wanted its want stands, marked put and held, and a held bus still takes its user-list revision (ADR 0539 §7).
|
||||||
|
put := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", Composed: dig(fNew)}
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
step string
|
||||||
|
wanted map[string]delivery.Want
|
||||||
|
want *delivery.Want // nil: nothing written on ace
|
||||||
|
}{
|
||||||
|
{"waiting for the operator", delivery.PutPerson, wantedOld(), nil},
|
||||||
|
{"wanted", delivery.PutWanted, wantedOld(), &delivery.Want{FP: fNew, Composed: dig(fNew), Put: true, Hold: true, Words: "put: the bus's user list"}},
|
||||||
|
{"holding, written already", delivery.PutHolding, map[string]delivery.Want{"ace": PutWant(put)}, nil},
|
||||||
|
{"released: the rules again", delivery.PutReleased, map[string]delivery.Want{"ace": PutWant(put)}, &delivery.Want{FP: fNew, Composed: dig(fNew)}},
|
||||||
|
} {
|
||||||
|
p := put
|
||||||
|
p.Step = c.step
|
||||||
|
m.Puts = map[string]delivery.Put{"ace": p}
|
||||||
|
out := Wants(m, n, c.wanted, nil)
|
||||||
|
got, wrote := out.Wants["ace"]
|
||||||
|
switch {
|
||||||
|
case c.want == nil && wrote:
|
||||||
|
t.Errorf("put %s: the rules wrote ace's want %+v", c.name, got)
|
||||||
|
case c.want != nil && (!wrote || !delivery.SameWant(normal(got), *c.want)):
|
||||||
|
t.Errorf("put %s: ace's want = %+v (written %v), want %+v", c.name, got, wrote, *c.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
userList := "sha256:" + strings.Repeat("7", 64)
|
||||||
|
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||||
|
return delivery.ComposedDigest{Digest: userList, Judgement: delivery.JudgedNeither, UserListOnly: true}, true
|
||||||
|
}
|
||||||
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
held := put
|
||||||
|
held.Step = delivery.PutHolding
|
||||||
|
m.Puts = map[string]delivery.Put{"ace": held}
|
||||||
|
if w := Wants(m, n, map[string]delivery.Want{"ace": PutWant(held)}, nil).Wants["ace"]; w.Composed != userList || !w.Put || !w.Hold {
|
||||||
|
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
||||||
|
}
|
||||||
|
m.Puts = nil
|
||||||
|
stepsOfAPut(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepsOfAPut takes a put through its steps (ADR 0546 clauses 3 to 8): the build, the controller's composition, the
|
||||||
|
// operator's yes, the bus's snapshot, the want, and the have.
|
||||||
|
func stepsOfAPut(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
now := time.Date(2026, 10, 25, 22, 0, 0, 0, time.UTC)
|
||||||
|
base := delivery.Put{FP: fNew, Hold: true, Why: "the bus's user list", At: now}.Said(now, delivery.PutAsked, "asked")
|
||||||
|
d := delivery.ComposedDigest{Digest: dig(fNew), Judgement: delivery.JudgedCannot, Words: "the have names no composition"}
|
||||||
|
running := delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
bus := func() PutWorld {
|
||||||
|
return PutWorld{Node: "novox", Module: "nats", Put: base, Assigned: true, Bus: true, Build: delivery.BuildDone, Newest: fNew,
|
||||||
|
Composed: &d, Granted: true, Have: &running, Now: now}
|
||||||
|
}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
world func() PutWorld
|
||||||
|
step string
|
||||||
|
says string
|
||||||
|
ask, snap bool
|
||||||
|
warranted bool
|
||||||
|
}{
|
||||||
|
{"not assigned", func() PutWorld { w := bus(); w.Assigned = false; return w }, delivery.PutRefused, "not assigned", false, false, false},
|
||||||
|
{"no build yet of the newest", func() PutWorld { w := bus(); w.Build = ""; return w }, delivery.PutBuild, "builds by itself", false, false, false},
|
||||||
|
{"no build of an older one", func() PutWorld { w := bus(); w.Build, w.Newest = "", fpOf('c'); return w }, delivery.PutRefused, "holds no build", false, false, false},
|
||||||
|
{"a failed build", func() PutWorld { w := bus(); w.Build, w.BuildWords = delivery.BuildFailed, "go vet"; return w }, delivery.PutRefused, "go vet", false, false, false},
|
||||||
|
{"no composition yet: waits", func() PutWorld { w := bus(); w.Composed = nil; return w }, delivery.PutCompose, "no composition of nats for novox", false, false, false},
|
||||||
|
{"composed, grant not naming the node", func() PutWorld { w := bus(); w.Granted = false; return w }, delivery.PutCompose, "does not name novox", false, false, false},
|
||||||
|
{"none within the bound: refused", func() PutWorld {
|
||||||
|
w := bus()
|
||||||
|
w.Composed = nil
|
||||||
|
w.Put = w.Put.Said(now.Add(-ComposeWithin-time.Second), delivery.PutCompose, "waits")
|
||||||
|
return w
|
||||||
|
}, delivery.PutRefused, "mesh-hand-compose", false, false, false},
|
||||||
|
{"the bus asks the operator", bus, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
||||||
|
{"the operator said no", func() PutWorld { w := bus(); w.Answer = OptionNo; return w }, delivery.PutRefused, "answered no", false, false, false},
|
||||||
|
{"the ask expired unanswered", func() PutWorld { w := bus(); w.AskExpired = true; return w }, delivery.PutRefused, "expired unanswered", false, false, false},
|
||||||
|
{"yes: the snapshot is asked", func() PutWorld { w := bus(); w.Answer = OptionYes; return w }, delivery.PutSnapshot, "asked novox's backup holder", false, true, false},
|
||||||
|
{"the snapshot is taken", func() PutWorld { w := bus(); w.Answer, w.Snapshotting = OptionYes, true; return w }, delivery.PutSnapshot, "waits for novox's backup holder", false, false, false},
|
||||||
|
{"the snapshot failed", func() PutWorld { w := bus(); w.Answer, w.SnapshotFailed = OptionYes, "no restic"; return w }, delivery.PutRefused, "no restic", false, false, false},
|
||||||
|
{"snapshotted: the want, warranted", func() PutWorld {
|
||||||
|
w := bus()
|
||||||
|
w.Answer, w.Snapshot = OptionYes, "novox's restore point"
|
||||||
|
w.Put.Ask, _ = delivery.RecordAskKey("novox", "nats", d.Digest)
|
||||||
|
return w
|
||||||
|
}, delivery.PutWanted, "held until released; the streams were snapshotted: novox's restore point", false, false, true},
|
||||||
|
{"a user-list revision of the bus needs nobody", func() PutWorld {
|
||||||
|
w := bus()
|
||||||
|
ul := d
|
||||||
|
ul.UserListOnly = true
|
||||||
|
w.Composed, w.Have = &ul, &delivery.Have{FP: fNew, Composed: dig(fOld), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
return w
|
||||||
|
}, delivery.PutWanted, "held until released", false, false, false},
|
||||||
|
{"a module not recorded needs nobody", func() PutWorld { w := bus(); w.Bus = false; return w }, delivery.PutWanted, "names", false, false, false},
|
||||||
|
{"a recorded module judged to recreate asks", func() PutWorld {
|
||||||
|
w := bus()
|
||||||
|
w.Bus, w.Recorded = false, true
|
||||||
|
r := d
|
||||||
|
r.Judgement = delivery.JudgedRecreates
|
||||||
|
w.Composed = &r
|
||||||
|
return w
|
||||||
|
}, delivery.PutPerson, "waits for the operator's yes", true, false, false},
|
||||||
|
{"wanted, the node runs it: holding", func() PutWorld {
|
||||||
|
w := bus()
|
||||||
|
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
||||||
|
w.Put.Composed = d.Digest
|
||||||
|
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
return w
|
||||||
|
}, delivery.PutHolding, "held there until released", false, false, false},
|
||||||
|
{"wanted without hold: done", func() PutWorld {
|
||||||
|
w := bus()
|
||||||
|
w.Put.Hold = false
|
||||||
|
w.Put = w.Put.Said(now, delivery.PutWanted, "wanted")
|
||||||
|
w.Put.Composed = d.Digest
|
||||||
|
w.Have = &delivery.Have{FP: fNew, Composed: d.Digest, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
return w
|
||||||
|
}, delivery.PutDone, "the rules write its want again", false, false, false},
|
||||||
|
{"released: nothing more", func() PutWorld { w := bus(); w.Put = w.Put.Said(now, delivery.PutReleased, "released"); return w }, delivery.PutReleased, "released", false, false, false},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
next := StepPut(c.world())
|
||||||
|
if next.Put.Step != c.step || !strings.Contains(next.Put.Words, c.says) {
|
||||||
|
t.Errorf("put, %s: at %s, %q; want %s saying %q", c.name, next.Put.Step, next.Put.Words, c.step, c.says)
|
||||||
|
}
|
||||||
|
if next.Ask != c.ask || next.Snapshot != c.snap || next.Warranted != c.warranted {
|
||||||
|
t.Errorf("put, %s: acts ask %v snapshot %v warranted %v; want %v %v %v", c.name, next.Ask, next.Snapshot, next.Warranted,
|
||||||
|
c.ask, c.snap, c.warranted)
|
||||||
|
}
|
||||||
|
if next.Ask && next.Put.Ask == "" {
|
||||||
|
t.Errorf("put, %s: an ask without its key", c.name)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
|
// 1906/D4 (ADR 0539 §16): the answer to a failed backup stays in the want while its target stays.
|
||||||
@@ -520,3 +701,343 @@ func TestAHaveFromBeforeTheSwitchGetsItsWantWithNoRecordHold(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1906/H3: a failed check whose node gave no words is held saying so, never with empty words.
|
||||||
|
func TestAHoldOfAFailureWithoutWordsSaysSo(t *testing.T) {
|
||||||
|
m, n := ready()
|
||||||
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||||
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}}, map[string]string{"ace": fOld})
|
||||||
|
if out.Held == nil || out.Held.Words != "health failed, phase running: the node gave no words" {
|
||||||
|
t.Fatalf("held = %+v", out.Held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/H2: given a held fingerprint that every node running the module now runs healthy, when the assigner passes, then the
|
||||||
|
// hold is lifted by itself and the wants move to it by the rules again. A node set back to another fingerprint, a node whose
|
||||||
|
// check fails, or a hold the operator answered "keep" leaves it held; a node that does not run the module at all does not count.
|
||||||
|
func TestAHoldIsLiftedOnceEveryNodeRunsItHealthy(t *testing.T) {
|
||||||
|
everywhere := func() (Module, Nodes) {
|
||||||
|
m, n := ready()
|
||||||
|
m.Held = true
|
||||||
|
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
return m, n
|
||||||
|
}
|
||||||
|
m, n := everywhere()
|
||||||
|
out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld})
|
||||||
|
if !out.Lifted || out.Held != nil {
|
||||||
|
t.Fatalf("lifted = %v, held = %+v: want the hold lifted", out.Lifted, out.Held)
|
||||||
|
}
|
||||||
|
for _, node := range []string{"ace", "g14"} {
|
||||||
|
if w := out.Wants[node]; w.FP != fNew || w.Rule != 0 {
|
||||||
|
t.Errorf("%s's want = %+v, want it moved to the lifted fingerprint", node, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
m, n = everywhere()
|
||||||
|
delete(n.Haves["g14"], "mail")
|
||||||
|
if out := Wants(m, n, wantedOld(), nil); !out.Lifted {
|
||||||
|
t.Error("a node that does not run the module kept the hold")
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, change := range map[string]func(*Module, *Nodes){
|
||||||
|
"set back": func(_ *Module, n *Nodes) {
|
||||||
|
n.Haves["g14"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
},
|
||||||
|
"failing": func(_ *Module, n *Nodes) {
|
||||||
|
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||||
|
},
|
||||||
|
"kept": func(m *Module, _ *Nodes) { m.Kept = true },
|
||||||
|
"no haves": func(_ *Module, n *Nodes) { delete(n.Haves["g14"], "mail"); delete(n.Haves["ace"], "mail") },
|
||||||
|
"not held": func(m *Module, _ *Nodes) { m.Held = false },
|
||||||
|
} {
|
||||||
|
m, n := everywhere()
|
||||||
|
change(&m, &n)
|
||||||
|
if out := Wants(m, n, wantedOld(), map[string]string{"ace": fOld, "g14": fOld}); out.Lifted {
|
||||||
|
t.Errorf("%s: the hold was lifted", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// x11 is screen-lock as on g14 and shanks on 2026-10-25 (issue 1945): built, granted and composed at F and at the older fOld
|
||||||
|
// on both, requiring x11-display, whose provider xorg runs on each node with its health unknown; each node runs screen-lock at fOld
|
||||||
|
// (health unknown) and holds no want for it. The canary g14 does not run F.
|
||||||
|
func x11() (Module, Nodes) {
|
||||||
|
m := Module{Name: "screen-lock", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"g14", "shanks"},
|
||||||
|
GrantedAt: func(fp string) bool { return fp == fNew || fp == fOld },
|
||||||
|
Manifest: catalogue.Manifest{Module: "screen-lock", Requires: []string{"x11-display"}}}
|
||||||
|
n := Nodes{
|
||||||
|
Online: map[string]bool{"g14": true, "shanks": true},
|
||||||
|
Haves: map[string]map[string]delivery.Have{
|
||||||
|
"g14": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
|
||||||
|
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
|
||||||
|
"shanks": {"screen-lock": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown},
|
||||||
|
"xorg": {FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}},
|
||||||
|
},
|
||||||
|
Providers: map[string]map[string][]string{"x11-display": {"xorg": {"g14", "shanks"}}},
|
||||||
|
}
|
||||||
|
return m, n
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC19: given a node that runs an older fingerprint of the new form, composed and granted, with no want, held back from F by
|
||||||
|
// the providers or the canary rule, when the assigner passes, then its first want names the fingerprint it runs with that
|
||||||
|
// composition, never the newest, and says the newest it is held back from; the canary is never passed over.
|
||||||
|
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
|
||||||
|
m, n := x11()
|
||||||
|
out := Wants(m, n, nil, nil)
|
||||||
|
for _, node := range []string{"g14", "shanks"} {
|
||||||
|
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
|
||||||
|
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(out.Neither) != 0 {
|
||||||
|
t.Errorf("left on neither list: %+v", out.Neither)
|
||||||
|
}
|
||||||
|
// with the provider healthy, only the canary holds shanks back: still at what it runs
|
||||||
|
m, n = x11()
|
||||||
|
n.Haves["g14"]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
n.Haves["shanks"]["xorg"] = n.Haves["g14"]["xorg"]
|
||||||
|
if w := Wants(m, n, nil, nil).Wants["shanks"]; w.FP != fOld || w.Rule != delivery.RuleCanary {
|
||||||
|
t.Errorf("shanks held back by the canary with no want: %+v, want at fOld by rule 3", w)
|
||||||
|
}
|
||||||
|
// a node already running F gets its want at F; a removed want is no want either
|
||||||
|
m, n = x11()
|
||||||
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||||
|
out = Wants(m, n, map[string]delivery.Want{"g14": {Removed: true}}, nil)
|
||||||
|
if w := out.Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) || w.Removed {
|
||||||
|
t.Errorf("g14 running F with a removed want = %+v, want F", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC19: a node running a fingerprint that is not of the new form, or one whose composition or grant does not exist, gets no
|
||||||
|
// want past the rules, and is named on neither list; the ordering rules still order every move from a want.
|
||||||
|
func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
|
||||||
|
for what, change := range map[string]func(*Module, *Nodes){
|
||||||
|
"an old-form build": func(_ *Module, n *Nodes) {
|
||||||
|
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||||
|
},
|
||||||
|
"no grant": func(m *Module, _ *Nodes) { m.GrantedAt = func(fp string) bool { return fp == fNew } },
|
||||||
|
"no composition": func(m *Module, _ *Nodes) {
|
||||||
|
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
||||||
|
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedNeither}, fp == fNew
|
||||||
|
}
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
m, n := x11()
|
||||||
|
change(&m, &n)
|
||||||
|
out := Wants(m, n, nil, nil)
|
||||||
|
if w, ok := out.Wants["shanks"]; ok {
|
||||||
|
t.Errorf("%s: shanks got a want: %+v", what, w)
|
||||||
|
}
|
||||||
|
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
|
||||||
|
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m, n := x11()
|
||||||
|
before := map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}, "shanks": {FP: fOld, Composed: dig(fOld)}}
|
||||||
|
out := Wants(m, n, before, nil)
|
||||||
|
if len(out.Neither) != 0 {
|
||||||
|
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
|
||||||
|
}
|
||||||
|
for node, w := range out.Wants {
|
||||||
|
if w.FP != fOld {
|
||||||
|
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC20: a new assignment (a node with no have of the module) waiting for the canary, the build or the grant gets no want and is
|
||||||
|
// never named on neither list: only a node that runs the module, or whose have names it, is; a module with no fingerprint of the new
|
||||||
|
// form goes with the node's own composition and is never named.
|
||||||
|
func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
|
||||||
|
m, n := x11()
|
||||||
|
delete(n.Haves["shanks"], "screen-lock")
|
||||||
|
out := Wants(m, n, nil, nil)
|
||||||
|
if w, ok := out.Wants["shanks"]; ok {
|
||||||
|
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
|
||||||
|
}
|
||||||
|
if _, ok := out.Neither["shanks"]; ok {
|
||||||
|
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
|
||||||
|
}
|
||||||
|
m.Built = false
|
||||||
|
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
|
||||||
|
t.Errorf("with no build: neither = %+v, want g14 alone (it runs the module), by rule 1", out.Neither)
|
||||||
|
}
|
||||||
|
m.FP = ""
|
||||||
|
if out := Wants(m, n, nil, nil); len(out.Neither) != 0 {
|
||||||
|
t.Errorf("a module of the node's own composition named on neither list: %+v", out.Neither)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
|
||||||
|
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
|
||||||
|
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
|
||||||
|
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
|
||||||
|
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
|
||||||
|
oldForm := func(n Nodes, node string) {
|
||||||
|
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||||
|
}
|
||||||
|
providersHealthy := func(n Nodes) {
|
||||||
|
for _, node := range []string{"g14", "shanks"} {
|
||||||
|
n.Haves[node]["xorg"] = delivery.Have{FP: fpOf('x'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// the canary g14 runs F healthy: shanks's first want names F
|
||||||
|
m, n := x11()
|
||||||
|
providersHealthy(n)
|
||||||
|
oldForm(n, "shanks")
|
||||||
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
out := Wants(m, n, nil, nil)
|
||||||
|
if w := out.Wants["shanks"]; w.FP != fNew || w.Composed != dig(fNew) || w.Newest != "" || w.Rule != 0 {
|
||||||
|
t.Errorf("canary healthy on F: shanks's want = %+v, want F", w)
|
||||||
|
}
|
||||||
|
if _, ok := out.Neither["shanks"]; ok {
|
||||||
|
t.Errorf("canary healthy on F: shanks still on neither list: %+v", out.Neither)
|
||||||
|
}
|
||||||
|
// the canary does not run F healthy yet: no want, and shanks stays on neither list by rule 3
|
||||||
|
m, n = x11()
|
||||||
|
providersHealthy(n)
|
||||||
|
oldForm(n, "shanks")
|
||||||
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||||
|
out = Wants(m, n, nil, nil)
|
||||||
|
if w, ok := out.Wants["shanks"]; ok {
|
||||||
|
t.Errorf("canary not healthy on F: shanks got a want: %+v", w)
|
||||||
|
}
|
||||||
|
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
|
||||||
|
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
|
||||||
|
}
|
||||||
|
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
|
||||||
|
m, n = x11()
|
||||||
|
oldForm(n, "shanks")
|
||||||
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
out = Wants(m, n, nil, nil)
|
||||||
|
if w, ok := out.Wants["shanks"]; ok {
|
||||||
|
t.Errorf("providers not healthy: shanks got a want: %+v", w)
|
||||||
|
}
|
||||||
|
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
|
||||||
|
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
|
||||||
|
}
|
||||||
|
// F not granted: nothing, even with the canary healthy
|
||||||
|
m, n = x11()
|
||||||
|
providersHealthy(n)
|
||||||
|
oldForm(n, "shanks")
|
||||||
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
m.Granted = false
|
||||||
|
if w, ok := Wants(m, n, nil, nil).Wants["shanks"]; ok {
|
||||||
|
t.Errorf("F not granted: shanks got a want: %+v", w)
|
||||||
|
}
|
||||||
|
// the old-form node is the canary itself: F once its providers run healthy
|
||||||
|
m, n = x11()
|
||||||
|
providersHealthy(n)
|
||||||
|
oldForm(n, "g14")
|
||||||
|
if w := Wants(m, n, nil, nil).Wants["g14"]; w.FP != fNew || w.Composed != dig(fNew) {
|
||||||
|
t.Errorf("old-form canary: g14's want = %+v, want F", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failed health counts only once the node is done moving: phase running or failed, never backing-up, applying or checking.
|
||||||
|
// The live row on novox at 15:27:36 on 2026-10-25 ("health failed, phase applying" at the controller's new fingerprint) held the
|
||||||
|
// canary two seconds before the new build was healthy.
|
||||||
|
func TestAFailedHealthWhileTheNodeMovesHoldsNothing(t *testing.T) {
|
||||||
|
for _, phase := range []delivery.Phase{delivery.PhaseBackingUp, delivery.PhaseApplying, delivery.PhaseChecking} {
|
||||||
|
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
||||||
|
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
|
||||||
|
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
|
||||||
|
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
|
||||||
|
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
|
||||||
|
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held != nil {
|
||||||
|
t.Errorf("phase %s, health failed: held %+v, want no hold while the node moves", phase, out.Held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, phase := range []delivery.Phase{delivery.PhaseRunning, delivery.PhaseFailed} {
|
||||||
|
m := Module{Name: "mesh-controller", FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
|
||||||
|
Manifest: catalogue.Manifest{Module: "mesh-controller"}}
|
||||||
|
n := Nodes{Online: map[string]bool{"novox": true}, Haves: map[string]map[string]delivery.Have{
|
||||||
|
"novox": {"mesh-controller": {FP: fNew, Phase: phase, Health: delivery.HealthFailed}}}}
|
||||||
|
before := map[string]delivery.Want{"novox": {FP: fNew, Composed: dig(fNew)}}
|
||||||
|
if out := Wants(m, n, before, map[string]string{"novox": fOld}); out.Held == nil || out.Held.Node != "novox" {
|
||||||
|
t.Errorf("phase %s, health failed: held %+v, want a hold on novox", phase, out.Held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A hold is lifted by itself once every node that runs the module runs F healthy: a have of a node that is not running the module
|
||||||
|
// (removed, held for its backup, or still moving) is not counted, so it does not keep the hold up for ever; a node running another
|
||||||
|
// fingerprint, or failed, still does.
|
||||||
|
func TestAHoldIsLiftedOverHavesThatAreNotRunning(t *testing.T) {
|
||||||
|
setup := func(phase delivery.Phase, fp string, health delivery.Health) (Module, Nodes) {
|
||||||
|
m := Module{Name: "web", FP: fNew, Built: true, Granted: true, Held: true, Composed: composedEverywhere, Nodes: []string{"ace", "g14"}}
|
||||||
|
n := Nodes{Online: map[string]bool{"ace": true, "g14": true}, Haves: map[string]map[string]delivery.Have{
|
||||||
|
"ace": {"web": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
|
||||||
|
"g14": {"web": {FP: fp, Phase: phase, Health: health}}}}
|
||||||
|
return m, n
|
||||||
|
}
|
||||||
|
for _, phase := range []delivery.Phase{delivery.PhaseRemoved, delivery.PhaseHeld, delivery.PhaseApplying, delivery.PhaseBackingUp, delivery.PhaseChecking} {
|
||||||
|
m, n := setup(phase, fOld, delivery.HealthUnknown)
|
||||||
|
if out := Wants(m, n, nil, nil); !out.Lifted {
|
||||||
|
t.Errorf("g14's have in phase %s kept the hold up", phase)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for what, h := range map[string]delivery.Have{
|
||||||
|
"running another fingerprint": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
|
||||||
|
"failed": {FP: fNew, Phase: delivery.PhaseFailed, Health: delivery.HealthFailed},
|
||||||
|
} {
|
||||||
|
m, n := setup(h.Phase, h.FP, h.Health)
|
||||||
|
if out := Wants(m, n, nil, nil); out.Lifted {
|
||||||
|
t.Errorf("g14 %s: the hold was lifted", what)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// a failed health on a removed have holds nothing
|
||||||
|
m, n := setup(delivery.PhaseRemoved, fNew, delivery.HealthFailed)
|
||||||
|
m.Held = false
|
||||||
|
if out := Wants(m, n, nil, nil); out.Held != nil {
|
||||||
|
t.Errorf("a removed have with a failed health held F: %+v", out.Held)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A module of policy record whose want was removed but which the node still runs gets its first want through the record ask: a
|
||||||
|
// removed want is no want, so a move the controller judges to recreate or move data waits for the operator's yes.
|
||||||
|
func TestAFirstWantAfterARemovedWantGoesThroughTheRecordAsk(t *testing.T) {
|
||||||
|
m, n := x11()
|
||||||
|
recording(&m)
|
||||||
|
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
|
||||||
|
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Policy: delivery.PolicyRecord}, true
|
||||||
|
}
|
||||||
|
out := Wants(m, n, map[string]delivery.Want{"shanks": {Removed: true}}, nil)
|
||||||
|
if w, ok := out.Wants["shanks"]; ok {
|
||||||
|
t.Errorf("shanks's first want after a removed want was written with no record ask: %+v", w)
|
||||||
|
}
|
||||||
|
asked := false
|
||||||
|
for _, r := range out.Record {
|
||||||
|
asked = asked || (r.Node == "shanks" && r.Digest == dig(fOld) && r.Judgement == delivery.JudgedRecreates)
|
||||||
|
}
|
||||||
|
if !asked {
|
||||||
|
t.Errorf("the record asks = %+v, want one for shanks at the composition it runs", out.Record)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1945/AC23: given a module held at F under rule 2 (the operator holds systemd-resolved on shanks, 2026-10-25), or held by the
|
||||||
|
// assigner in this pass after a failed health check of F, when a node that runs it holds no want for it, then the node is not on
|
||||||
|
// neither list: a held module is on the held list, whose hold is said and answered on its own.
|
||||||
|
func TestAHeldModuleWithNoWantIsNotOnNeitherList(t *testing.T) {
|
||||||
|
for what, change := range map[string]func(*Module, *Nodes){
|
||||||
|
"held at F": func(m *Module, _ *Nodes) { m.Held, m.Kept = true, true },
|
||||||
|
"held at F, running an old-form build": func(m *Module, n *Nodes) {
|
||||||
|
m.Held, m.Kept = true, true
|
||||||
|
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||||
|
},
|
||||||
|
"held by this pass": func(_ *Module, n *Nodes) {
|
||||||
|
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||||
|
n.Haves["shanks"]["screen-lock"] = delivery.Have{FP: "build-2", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
m, n := x11()
|
||||||
|
change(&m, &n)
|
||||||
|
out := Wants(m, n, nil, nil)
|
||||||
|
if len(out.Neither) != 0 {
|
||||||
|
t.Errorf("%s: a held module is on neither list: %+v", what, out.Neither)
|
||||||
|
}
|
||||||
|
if out.Rules["shanks"].Rule != delivery.RuleNotHeld {
|
||||||
|
t.Errorf("%s: shanks is held back by %+v, want rule 2", what, out.Rules["shanks"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ func load(t *testing.T) catalogue.Manifest {
|
|||||||
|
|
||||||
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
|
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
|
||||||
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
|
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
|
||||||
// its four states (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
// the states it declares (the differences kept read-only, 1906/D5), and holds a credential of its own.
|
||||||
func TestManifest(t *testing.T) {
|
func TestManifest(t *testing.T) {
|
||||||
m := load(t)
|
m := load(t)
|
||||||
if m.Module != "mesh-delivery-assigner" {
|
if m.Module != "mesh-delivery-assigner" {
|
||||||
@@ -42,7 +42,7 @@ func TestManifest(t *testing.T) {
|
|||||||
for _, s := range m.State {
|
for _, s := range m.State {
|
||||||
states[s.Name] = s.History
|
states[s.Name] = s.History
|
||||||
}
|
}
|
||||||
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "wants": 20}; !reflect.DeepEqual(states, want) {
|
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||||
t.Errorf("state = %v, want %v", states, want)
|
t.Errorf("state = %v, want %v", states, want)
|
||||||
}
|
}
|
||||||
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||||
@@ -56,7 +56,16 @@ func TestManifest(t *testing.T) {
|
|||||||
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
if !slices.Equal(m.Uses, []string{"node-build-go", "operator-channel"}) {
|
||||||
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
|
||||||
}
|
}
|
||||||
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
|
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
|
||||||
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
|
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
|
||||||
|
// calls the controller's warranted and the bus node's backup holder.
|
||||||
|
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
|
||||||
|
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
|
||||||
|
}
|
||||||
|
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
|
||||||
|
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||||
|
}
|
||||||
|
if !slices.Equal(m.Emits, []string{"put"}) {
|
||||||
|
t.Errorf("emits = %v, want put", m.Emits)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,10 +20,24 @@
|
|||||||
"node-build-go",
|
"node-build-go",
|
||||||
"operator-channel"
|
"operator-channel"
|
||||||
],
|
],
|
||||||
|
"invokes": [
|
||||||
|
"seat:mesh-controller.warranted",
|
||||||
|
"seat:node-backup.now",
|
||||||
|
"seat:node-backup.backed-up"
|
||||||
|
],
|
||||||
|
"emits": [
|
||||||
|
"put"
|
||||||
|
],
|
||||||
|
"tools": [
|
||||||
|
"assigner_put",
|
||||||
|
"assigner_release",
|
||||||
|
"assigner_puts"
|
||||||
|
],
|
||||||
"state": [
|
"state": [
|
||||||
{ "name": "asks", "history": 5 },
|
{ "name": "asks", "history": 5 },
|
||||||
{ "name": "compositions", "history": 5 },
|
{ "name": "compositions", "history": 5 },
|
||||||
{ "name": "differences", "history": 64 },
|
{ "name": "differences", "history": 64 },
|
||||||
|
{ "name": "puts", "history": 20 },
|
||||||
{ "name": "wants", "history": 20 }
|
{ "name": "wants", "history": 20 }
|
||||||
],
|
],
|
||||||
"own-secrets": {
|
"own-secrets": {
|
||||||
@@ -71,7 +85,10 @@
|
|||||||
"language": "go",
|
"language": "go",
|
||||||
"system": "arch",
|
"system": "arch",
|
||||||
"from": "cmd/mesh-delivery-assigner",
|
"from": "cmd/mesh-delivery-assigner",
|
||||||
"binary": "mesh-delivery-assigner"
|
"binary": "mesh-delivery-assigner",
|
||||||
|
"loads": [
|
||||||
|
"mesh-delivery-assigner"
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -66,6 +66,11 @@ func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyV
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return settled(ctx, w, each)
|
||||||
|
}
|
||||||
|
|
||||||
|
// settled hands each entry the watch delivers before its end-of-initial marker to each, and answers the watch.
|
||||||
|
func settled(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
|
|||||||
@@ -162,11 +162,29 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// trunkHead is the trunk head of novox/mesh-catalog the bed's source revision names: newer than the module values' commit.
|
||||||
|
const trunkHead = "3333333333333333333333333333333333333333"
|
||||||
|
|
||||||
|
// trunk writes the gate's source.main naming head as novox/mesh-catalog's trunk head, and answers its revision.
|
||||||
|
func (b *bed) trunk(t *testing.T, head string) uint64 {
|
||||||
|
t.Helper()
|
||||||
|
data, err := delivery.EncodeSource(delivery.Source{V: delivery.Version, Group: "g", Heads: map[string]string{"novox/mesh-catalog": head}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rev, err := b.source.Put(b.ctx, delivery.SourceMainKey, data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return rev
|
||||||
|
}
|
||||||
|
|
||||||
type bed struct {
|
type bed struct {
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
js jetstream.JetStream
|
js jetstream.JetStream
|
||||||
modules jetstream.KeyValue
|
modules jetstream.KeyValue
|
||||||
candidates jetstream.KeyValue
|
candidates jetstream.KeyValue
|
||||||
|
source jetstream.KeyValue
|
||||||
builds jetstream.KeyValue
|
builds jetstream.KeyValue
|
||||||
tools map[string]jetstream.KeyValue
|
tools map[string]jetstream.KeyValue
|
||||||
instances jetstream.KeyValue
|
instances jetstream.KeyValue
|
||||||
@@ -243,6 +261,7 @@ func startBedWith(t *testing.T, closed bool) *bed {
|
|||||||
}
|
}
|
||||||
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
|
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
|
||||||
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
|
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
|
||||||
|
b.source = mk(delivery.SourceBucket, delivery.SourceHistory)
|
||||||
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
||||||
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
||||||
b.instances = mk(instancesBucket, 1)
|
b.instances = mk(instancesBucket, 1)
|
||||||
@@ -513,7 +532,9 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
|
|||||||
|
|
||||||
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
|
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
|
||||||
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
|
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
|
||||||
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk commit (issue 1684).
|
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk head of the candidate's source
|
||||||
|
// revision, newer than the module value's commit (issue 1684; the operator 2026-10-25: a value names the commit its
|
||||||
|
// fingerprint first appeared at).
|
||||||
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
|
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
|
||||||
b := startBed(t)
|
b := startBed(t)
|
||||||
other := "src2:" + strings.Repeat("e", 64)
|
other := "src2:" + strings.Repeat("e", 64)
|
||||||
@@ -530,7 +551,7 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
|||||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"},
|
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"},
|
||||||
Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}}
|
Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}}
|
||||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -550,12 +571,12 @@ func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *tes
|
|||||||
defer b.fake.mu.Unlock()
|
defer b.fake.mu.Unlock()
|
||||||
found := false
|
found := false
|
||||||
for _, f := range b.fake.fetches {
|
for _, f := range b.fake.fetches {
|
||||||
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == commit {
|
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == trunkHead {
|
||||||
found = true
|
found = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if !found {
|
if !found {
|
||||||
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk commit", b.fake.fetches)
|
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk head it was previewed onto, not the module value's commit", b.fake.fetches)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -603,7 +624,7 @@ func TestAnImageIsBuiltOnOneNodeAndOnlyATrunkKeyPublishes(t *testing.T) { // 170
|
|||||||
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"imaged"},
|
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"imaged"},
|
||||||
Fps: map[string]delivery.CandidateModule{"imaged": {FP2: fp, Manifest: digest}}}}
|
Fps: map[string]delivery.CandidateModule{"imaged": {FP2: fp, Manifest: digest}}}}
|
||||||
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
|
||||||
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", Base: b.trunk(t, trunkHead), HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,9 +16,12 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
|
"maps"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -174,6 +177,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
Logf: logger.Printf,
|
Logf: logger.Printf,
|
||||||
Stored: storedVia(calls.Call),
|
Stored: storedVia(calls.Call),
|
||||||
}
|
}
|
||||||
|
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||||
|
// revision (the gate's source state). Where the bucket is not there yet no candidate's fingerprint is built, and said.
|
||||||
|
// The source's revisions are watched with their history (a direct get of one revision is not what a read grants): a
|
||||||
|
// revision heard after the candidate that names it builds that candidate then.
|
||||||
|
trunks := &sourceHeads{}
|
||||||
|
var sw jetstream.KeyWatcher
|
||||||
|
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
||||||
|
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||||
|
logger.Printf("the bucket %s does not exist yet: no candidate's fingerprint is built until the builder starts again", delivery.SourceBucket)
|
||||||
|
case err != nil:
|
||||||
|
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||||
|
default:
|
||||||
|
w, err := kv.Watch(ctx, delivery.SourceMainKey, jetstream.IncludeHistory())
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the bucket %s cannot be watched: %w", delivery.SourceBucket, err)
|
||||||
|
}
|
||||||
|
if sw, err = settled(ctx, w, func(e jetstream.KeyValueEntry) { trunks.put(e) }); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.Trunk = trunks.head
|
||||||
|
}
|
||||||
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
|
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
|
||||||
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
|
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
|
||||||
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
|
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
|
||||||
@@ -267,9 +291,18 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
logger.Printf("started on %s", cred.Node)
|
logger.Printf("started on %s", cred.Node)
|
||||||
errs := make(chan error, 2)
|
errs := make(chan error, 3)
|
||||||
go func() { errs <- follow(ctx, w, each) }()
|
go func() { errs <- follow(ctx, w, each) }()
|
||||||
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
go func() { errs <- follow(ctx, cw, eachCandidate) }()
|
||||||
|
if sw != nil {
|
||||||
|
go func() {
|
||||||
|
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
|
||||||
|
if trunks.put(e) {
|
||||||
|
b.TrunkMoved(ctx)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
}
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
b.Wait()
|
b.Wait()
|
||||||
@@ -282,3 +315,50 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sourceHeads are the trunk heads of each revision of the gate's source.main the watch delivered, the newest
|
||||||
|
// delivery.SourceHistory of them: what a candidate's base revision names.
|
||||||
|
type sourceHeads struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
byRev map[uint64]map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// put takes one revision in and answers whether it is new.
|
||||||
|
func (s *sourceHeads) put(e jetstream.KeyValueEntry) bool {
|
||||||
|
if !opOf(e) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
src, err := delivery.DecodeSource(e.Value())
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if s.byRev == nil {
|
||||||
|
s.byRev = map[uint64]map[string]string{}
|
||||||
|
}
|
||||||
|
if _, had := s.byRev[e.Revision()]; had {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
s.byRev[e.Revision()] = src.Heads
|
||||||
|
for len(s.byRev) > delivery.SourceHistory {
|
||||||
|
delete(s.byRev, slices.Min(slices.Collect(maps.Keys(s.byRev))))
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// head is the head of repository in revision rev; an error when the revision is not heard (yet, or any more).
|
||||||
|
func (s *sourceHeads) head(_ context.Context, rev uint64, repository string) (string, error) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
heads, ok := s.byRev[rev]
|
||||||
|
if !ok {
|
||||||
|
return "", fmt.Errorf("source revision %d is not heard (it is built once it is)", rev)
|
||||||
|
}
|
||||||
|
for r, head := range heads {
|
||||||
|
if delivery.SameRepository(r, repository) {
|
||||||
|
return head, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -28,8 +28,11 @@ import (
|
|||||||
//
|
//
|
||||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||||
|
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||||
|
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||||
|
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||||
type toolchainGate struct {
|
type toolchainGate struct {
|
||||||
@@ -38,14 +41,14 @@ type toolchainGate struct {
|
|||||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||||
logf func(format string, a ...any)
|
logf func(format string, a ...any)
|
||||||
known bool // the assignments are settled
|
known bool // the assignments are settled
|
||||||
standing map[string]bool // <node>.<module> with an open assignment
|
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||||
}
|
}
|
||||||
|
|
||||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
g.modules[m] = true
|
g.modules[m] = true
|
||||||
}
|
}
|
||||||
@@ -67,20 +70,31 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
|||||||
return g.pass(module, key)
|
return g.pass(module, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
// whether what it passed on changed.
|
||||||
|
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||||
node, module, ok := strings.Cut(key, ".")
|
node, module, ok := strings.Cut(key, ".")
|
||||||
if !ok || !g.modules[module] {
|
if !ok || !g.modules[module] {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
open := !deleted && openAssignment(value)
|
assigned, open := openAssignment(value)
|
||||||
|
open = open && !deleted
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
if g.standing[key] == open {
|
was, stood := g.standing[key]
|
||||||
|
since := assigned
|
||||||
|
if since.IsZero() {
|
||||||
|
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||||
|
since = at
|
||||||
|
if stood {
|
||||||
|
since = was
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if open == stood && (!open || since.Equal(was)) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if open {
|
if open {
|
||||||
g.standing[key] = true
|
g.standing[key] = since
|
||||||
} else {
|
} else {
|
||||||
delete(g.standing, key)
|
delete(g.standing, key)
|
||||||
}
|
}
|
||||||
@@ -111,37 +125,58 @@ func (g *toolchainGate) Settled() bool {
|
|||||||
return changed
|
return changed
|
||||||
}
|
}
|
||||||
|
|
||||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||||
// otherwise. The caller holds mu.
|
// known), deleted otherwise. The caller holds mu.
|
||||||
func (g *toolchainGate) pass(module, key string) bool {
|
func (g *toolchainGate) pass(module, key string) bool {
|
||||||
value, held := g.raw[module][key]
|
value, held := g.raw[module][key]
|
||||||
if !held {
|
if !held {
|
||||||
return g.forward(module, key, nil, true)
|
return g.forward(module, key, nil, true)
|
||||||
}
|
}
|
||||||
node, _, err := delivery.ParseToolchainKey(key)
|
node, _, err := delivery.ParseToolchainKey(key)
|
||||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
why := ""
|
||||||
|
if err == nil && g.known {
|
||||||
|
why = g.leftOut(module, node, value)
|
||||||
|
}
|
||||||
|
if why == "" {
|
||||||
delete(g.left, module+"/"+key)
|
delete(g.left, module+"/"+key)
|
||||||
return g.forward(module, key, value, false)
|
return g.forward(module, key, value, false)
|
||||||
}
|
}
|
||||||
if !g.left[module+"/"+key] {
|
if g.left[module+"/"+key] != why {
|
||||||
g.left[module+"/"+key] = true
|
g.left[module+"/"+key] = why
|
||||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||||
delivery.ToolchainsBucket(module), key, module, node)
|
|
||||||
}
|
}
|
||||||
return g.forward(module, key, nil, true)
|
return g.forward(module, key, nil, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||||
func openAssignment(value []byte) bool {
|
// The caller holds mu.
|
||||||
|
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||||
|
since, open := g.standing[node+"."+module]
|
||||||
|
if !open {
|
||||||
|
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||||
|
}
|
||||||
|
tc, err := delivery.DecodeToolchain(value)
|
||||||
|
if err == nil && tc.At.Before(since) {
|
||||||
|
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||||
|
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||||
|
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||||
|
// it was assigned, zero where the value does not say.
|
||||||
|
func openAssignment(value []byte) (time.Time, bool) {
|
||||||
var a struct {
|
var a struct {
|
||||||
Ended time.Time `json:"ended"`
|
Assigned time.Time `json:"assigned"`
|
||||||
State string `json:"state"`
|
Ended time.Time `json:"ended"`
|
||||||
|
State string `json:"state"`
|
||||||
}
|
}
|
||||||
if json.Unmarshal(value, &a) != nil {
|
if json.Unmarshal(value, &a) != nil {
|
||||||
return false
|
return time.Time{}, false
|
||||||
}
|
}
|
||||||
return a.Ended.IsZero() && a.State == ""
|
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||||
@@ -155,7 +190,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
|||||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||||
return errs, nil
|
return errs, nil
|
||||||
}
|
}
|
||||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||||
w, err := settle(ctx, kv, each)
|
w, err := settle(ctx, kv, each)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||||
@@ -165,7 +200,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
|||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||||
changed(ctx)
|
changed(ctx)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -44,10 +45,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||||
}
|
}
|
||||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||||
}
|
}
|
||||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||||
if !gate.Settled() {
|
if !gate.Settled() {
|
||||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||||
}
|
}
|
||||||
@@ -63,20 +64,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
t.Fatal("an unassigned node's report changed what was passed on")
|
t.Fatal("an unassigned node's report changed what was passed on")
|
||||||
}
|
}
|
||||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||||
t.Fatal("assigning ace again passed nothing on")
|
t.Fatal("assigning ace again passed nothing on")
|
||||||
}
|
}
|
||||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||||
}
|
}
|
||||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||||
t.Fatal("a pending assignment is no assignment on the machine")
|
t.Fatal("a pending assignment is no assignment on the machine")
|
||||||
}
|
}
|
||||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||||
t.Fatalf("got %q, %q", v, none)
|
t.Fatalf("got %q, %q", v, none)
|
||||||
}
|
}
|
||||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||||
t.Fatalf("got %q", v)
|
t.Fatalf("got %q", v)
|
||||||
}
|
}
|
||||||
@@ -90,23 +91,75 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||||
|
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||||
|
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||||
|
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||||
set := &buildrun.ToolchainSet{}
|
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||||
gate.Settled()
|
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
key, _ := gateReport(t, node, version)
|
||||||
t.Fatal("an assignment with no report passed something on")
|
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return key, value
|
||||||
}
|
}
|
||||||
key, value := gateReport(t, "g14", "v26.12.1")
|
for _, c := range []struct {
|
||||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
name string
|
||||||
t.Fatal("an assigned node's report was not passed on")
|
assignment string // ace's assignment value
|
||||||
}
|
revision time.Time // when the bus wrote it
|
||||||
key, value = gateReport(t, "ace", "v26.12.0")
|
at time.Time // when ace wrote its report
|
||||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
counts bool
|
||||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
}{
|
||||||
}
|
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||||
t.Fatalf("got %q, %q", v, none)
|
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||||
|
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
set := &buildrun.ToolchainSet{}
|
||||||
|
var said []string
|
||||||
|
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||||
|
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||||
|
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||||
|
key, value := gateReport(t, node, "v26.12.1")
|
||||||
|
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||||
|
}
|
||||||
|
gate.Settled()
|
||||||
|
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||||
|
t.Fatal("an assignment with no report passed something on")
|
||||||
|
}
|
||||||
|
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||||
|
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||||
|
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||||
|
}
|
||||||
|
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||||
|
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||||
|
t.Fatal("the same assignment written again changed what was passed on")
|
||||||
|
}
|
||||||
|
v, none := agreedOf(set)
|
||||||
|
if c.counts {
|
||||||
|
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||||
|
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if v != "v26.12.1" || none != "" {
|
||||||
|
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||||
|
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||||
|
}
|
||||||
|
// ace writes its report again since it was assigned: it counts.
|
||||||
|
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||||
|
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||||
|
t.Fatal("ace's new report was not passed on")
|
||||||
|
}
|
||||||
|
if n := len(set.Entries("typescript")); n != 4 {
|
||||||
|
t.Fatalf("four builders report; got %d", n)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -73,9 +73,17 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
|
|||||||
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
|
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
|
||||||
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
|
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
|
||||||
}
|
}
|
||||||
|
// The preview is merged onto the trunk head the candidate was previewed onto: the head of the pull request's
|
||||||
|
// repository in the gate's source revision c.Base (ADR 0484 §3). Never the module's value's commit: that is the
|
||||||
|
// commit its fingerprint first appeared at, and may be many merges behind the trunk.
|
||||||
|
base, err := b.trunkHead(ctx, c.Base, p.Repository)
|
||||||
|
if err != nil {
|
||||||
|
b.logf("candidates.%s: the module %s's fingerprint %s is not built: %v", c.Group, name, cm.FP2, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
if _, onTrunk := b.trunkFP[key]; !onTrunk {
|
if _, onTrunk := b.trunkFP[key]; !onTrunk {
|
||||||
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit}
|
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: base}
|
||||||
}
|
}
|
||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
b.consider(ctx, name, m)
|
b.consider(ctx, name, m)
|
||||||
@@ -106,3 +114,18 @@ func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]del
|
|||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// trunkHead is the head of repository in the gate's source revision rev (Builder.Trunk), or why it cannot be said.
|
||||||
|
func (b *Builder) trunkHead(ctx context.Context, rev uint64, repository string) (string, error) {
|
||||||
|
if b.Trunk == nil {
|
||||||
|
return "", fmt.Errorf("the gate's source cannot be read here, so the trunk head %s was previewed onto is not known", repository)
|
||||||
|
}
|
||||||
|
head, err := b.Trunk(ctx, rev, repository)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("the trunk head of %s at source revision %d cannot be read: %w", repository, rev, err)
|
||||||
|
}
|
||||||
|
if head == "" {
|
||||||
|
return "", fmt.Errorf("source revision %d names no trunk head of %s", rev, repository)
|
||||||
|
}
|
||||||
|
return head, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,9 +2,13 @@ package build
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -40,11 +44,74 @@ func candidateValue(t *testing.T, group string, number int, head string) []byte
|
|||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
headX = "1111111111111111111111111111111111111111"
|
// trunkHead is the trunk head the rig's source revision names: newer than the module value's commit, which stays at the
|
||||||
headY = "2222222222222222222222222222222222222222"
|
// commit its fingerprint first appeared at.
|
||||||
|
trunkHead = "3333333333333333333333333333333333333333"
|
||||||
|
headX = "1111111111111111111111111111111111111111"
|
||||||
|
headY = "2222222222222222222222222222222222222222"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// indexerCandidate is a candidate as the indexer writes it, with the gate's source revision it names (mesh-delivery-indexer
|
||||||
|
// internal/index testdata/candidate-written.json, kept equal to what the indexer writes by its own test).
|
||||||
|
func indexerCandidate(t *testing.T) (rev uint64, heads map[string]string, c delivery.Candidate, raw []byte) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile("../../../mesh-delivery-indexer/internal/index/testdata/candidate-written.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var w struct {
|
||||||
|
Source struct {
|
||||||
|
Rev uint64 `json:"rev"`
|
||||||
|
Heads map[string]string `json:"heads"`
|
||||||
|
} `json:"source"`
|
||||||
|
Candidate json.RawMessage `json:"candidate"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(data, &w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if c, err = delivery.DecodeCandidate(w.Candidate); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return w.Source.Rev, w.Source.Heads, c, w.Candidate
|
||||||
|
}
|
||||||
|
|
||||||
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
|
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
|
||||||
|
// The candidate the indexer writes is merged onto the trunk head of the source revision it names, never onto the module
|
||||||
|
// value's commit, which stays at the commit its fingerprint first appeared at (review of the indexer writing on change).
|
||||||
|
{
|
||||||
|
rev, heads, c, raw := indexerCandidate(t)
|
||||||
|
p := c.Pulls[0]
|
||||||
|
cm := p.Fps["avahi"]
|
||||||
|
ir := newRig(t, goBundle)
|
||||||
|
ir.mf.byDigest[cm.Manifest] = catalogue.Manifest{Module: "avahi", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}}
|
||||||
|
ir.b.Trunk = func(_ context.Context, at uint64, repository string) (string, error) {
|
||||||
|
if at != rev {
|
||||||
|
return "", fmt.Errorf("source revision %d is not heard", at)
|
||||||
|
}
|
||||||
|
return heads[repository], nil
|
||||||
|
}
|
||||||
|
const valueCommit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
|
||||||
|
value, err := delivery.EncodeModule(delivery.Module{Repository: p.Repository, Commit: valueCommit, FP2: "src2:" + hex64('e'),
|
||||||
|
Manifest: cm.Manifest, Path: "modules/avahi", Toolchains: agreedToolchains})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ir.b.Module(context.Background(), "avahi", value)
|
||||||
|
ir.b.Wait()
|
||||||
|
ir.bm.mu.Lock()
|
||||||
|
ir.bm.calls = nil
|
||||||
|
ir.bm.mu.Unlock()
|
||||||
|
ir.b.Candidate(context.Background(), "feat%2Fx", raw)
|
||||||
|
ir.b.Wait()
|
||||||
|
fetches := ir.bm.callsOf("build_go_fetch")
|
||||||
|
if len(fetches) != 1 {
|
||||||
|
t.Fatalf("%d fetches of the indexer's candidate, want one", len(fetches))
|
||||||
|
}
|
||||||
|
f := fetches[0].body.(map[string]any)
|
||||||
|
if want := heads[p.Repository]; want == "" || f["base"] != want || f["base"] == valueCommit || f["commit"] != p.Head.Head {
|
||||||
|
t.Errorf("fetch args = %+v, want the head merged onto the trunk head %.8s of source revision %d", f, want, rev)
|
||||||
|
}
|
||||||
|
}
|
||||||
r := newRig(t, goBundle)
|
r := newRig(t, goBundle)
|
||||||
trunkModule(t, r)
|
trunkModule(t, r)
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
||||||
@@ -58,8 +125,8 @@ func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoThe
|
|||||||
t.Fatalf("%d fetches, want one", len(fetches))
|
t.Fatalf("%d fetches, want one", len(fetches))
|
||||||
}
|
}
|
||||||
f := fetches[0].body.(map[string]any)
|
f := fetches[0].body.(map[string]any)
|
||||||
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" {
|
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != trunkHead || f["repository"] != "novox/mesh-catalog" {
|
||||||
t.Errorf("fetch args = %+v, want the head, its ref and the trunk commit to merge onto", f)
|
t.Errorf("fetch args = %+v, want the head, its ref and the trunk head the candidate was previewed onto", f)
|
||||||
}
|
}
|
||||||
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
|
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
|
||||||
t.Errorf("build args = %+v", b)
|
t.Errorf("build args = %+v", b)
|
||||||
|
|||||||
@@ -379,8 +379,9 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A
|
|||||||
t.Fatal("no fetch")
|
t.Fatal("no fetch")
|
||||||
}
|
}
|
||||||
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
|
req, err := buildstep.ParseFetchRequest(f[0].body.(map[string]any))
|
||||||
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base == "" {
|
// The base is the trunk head the candidate was previewed onto, never the module value's commit (an older one).
|
||||||
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
|
if err != nil || req.Dir != "modules/m" || strings.Join(req.Paths, ",") != "package.json,src" || req.Base != trunkHead {
|
||||||
|
t.Errorf("fetch request = %+v, %v; want the trunk head %.8s as the base, the module's directory and its named paths", req, err, trunkHead)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -53,6 +53,9 @@ type Builder struct {
|
|||||||
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
|
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
|
||||||
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
||||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||||
|
// Trunk answers the head of a repository in the gate's source revision rev (source.main at that revision): the trunk a
|
||||||
|
// candidate's preview is merged onto (Candidate.Base). Nil builds no candidate's fingerprint, and says why.
|
||||||
|
Trunk func(ctx context.Context, rev uint64, repository string) (string, error)
|
||||||
|
|
||||||
once sync.Once
|
once sync.Once
|
||||||
sem chan struct{}
|
sem chan struct{}
|
||||||
@@ -269,6 +272,13 @@ func (b *Builder) Reconsider(ctx context.Context) {
|
|||||||
for _, k := range sortedKeys(known) {
|
for _, k := range sortedKeys(known) {
|
||||||
b.consider(ctx, k, known[k])
|
b.consider(ctx, k, known[k])
|
||||||
}
|
}
|
||||||
|
b.TrunkMoved(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TrunkMoved considers every candidate again once a revision of the gate's source is heard: a candidate heard before the
|
||||||
|
// revision it was previewed onto had no trunk head to be merged onto.
|
||||||
|
func (b *Builder) TrunkMoved(ctx context.Context) {
|
||||||
|
b.init()
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
cands := make([]delivery.Candidate, 0, len(b.cands))
|
cands := make([]delivery.Candidate, 0, len(b.cands))
|
||||||
for _, c := range b.cands {
|
for _, c := range b.cands {
|
||||||
|
|||||||
@@ -176,6 +176,7 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
|
|||||||
r.b = &Builder{
|
r.b = &Builder{
|
||||||
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
|
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
|
||||||
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
|
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
|
||||||
|
Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil },
|
||||||
}
|
}
|
||||||
var err error
|
var err error
|
||||||
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
|
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
|||||||
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||||
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
|
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
|
||||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||||
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||||
}
|
}
|
||||||
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||||
|
|||||||
@@ -21,6 +21,7 @@
|
|||||||
"mesh-build-node.steps",
|
"mesh-build-node.steps",
|
||||||
"mesh-build-node.toolchains",
|
"mesh-build-node.toolchains",
|
||||||
"mesh-controller.assignments",
|
"mesh-controller.assignments",
|
||||||
|
"mesh-delivery-gate.source",
|
||||||
"mesh-delivery-indexer.candidates",
|
"mesh-delivery-indexer.candidates",
|
||||||
"mesh-delivery-indexer.manifests",
|
"mesh-delivery-indexer.manifests",
|
||||||
"mesh-delivery-indexer.modules"
|
"mesh-delivery-indexer.modules"
|
||||||
|
|||||||
@@ -182,7 +182,8 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu
|
|||||||
return b, err == nil, err
|
return b, err == nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// CommitOf is the commit the indexer's record of the module was computed at: the commit of its repository the mesh runs.
|
// CommitOf is the commit the indexer's record of the module names: the commit its fingerprint first appeared at, so the source
|
||||||
|
// of the build the mesh runs (a later merge that leaves the module alone writes no record, the operator 2026-10-25).
|
||||||
func (r *repoModules) CommitOf(module string) string {
|
func (r *repoModules) CommitOf(module string) string {
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
|
|||||||
@@ -28,8 +28,11 @@ import (
|
|||||||
//
|
//
|
||||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||||
|
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||||
|
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||||
|
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||||
type toolchainGate struct {
|
type toolchainGate struct {
|
||||||
@@ -38,14 +41,14 @@ type toolchainGate struct {
|
|||||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||||
logf func(format string, a ...any)
|
logf func(format string, a ...any)
|
||||||
known bool // the assignments are settled
|
known bool // the assignments are settled
|
||||||
standing map[string]bool // <node>.<module> with an open assignment
|
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||||
}
|
}
|
||||||
|
|
||||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
g.modules[m] = true
|
g.modules[m] = true
|
||||||
}
|
}
|
||||||
@@ -67,20 +70,31 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
|||||||
return g.pass(module, key)
|
return g.pass(module, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
// whether what it passed on changed.
|
||||||
|
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||||
node, module, ok := strings.Cut(key, ".")
|
node, module, ok := strings.Cut(key, ".")
|
||||||
if !ok || !g.modules[module] {
|
if !ok || !g.modules[module] {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
open := !deleted && openAssignment(value)
|
assigned, open := openAssignment(value)
|
||||||
|
open = open && !deleted
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
if g.standing[key] == open {
|
was, stood := g.standing[key]
|
||||||
|
since := assigned
|
||||||
|
if since.IsZero() {
|
||||||
|
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||||
|
since = at
|
||||||
|
if stood {
|
||||||
|
since = was
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if open == stood && (!open || since.Equal(was)) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if open {
|
if open {
|
||||||
g.standing[key] = true
|
g.standing[key] = since
|
||||||
} else {
|
} else {
|
||||||
delete(g.standing, key)
|
delete(g.standing, key)
|
||||||
}
|
}
|
||||||
@@ -111,37 +125,58 @@ func (g *toolchainGate) Settled() bool {
|
|||||||
return changed
|
return changed
|
||||||
}
|
}
|
||||||
|
|
||||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||||
// otherwise. The caller holds mu.
|
// known), deleted otherwise. The caller holds mu.
|
||||||
func (g *toolchainGate) pass(module, key string) bool {
|
func (g *toolchainGate) pass(module, key string) bool {
|
||||||
value, held := g.raw[module][key]
|
value, held := g.raw[module][key]
|
||||||
if !held {
|
if !held {
|
||||||
return g.forward(module, key, nil, true)
|
return g.forward(module, key, nil, true)
|
||||||
}
|
}
|
||||||
node, _, err := delivery.ParseToolchainKey(key)
|
node, _, err := delivery.ParseToolchainKey(key)
|
||||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
why := ""
|
||||||
|
if err == nil && g.known {
|
||||||
|
why = g.leftOut(module, node, value)
|
||||||
|
}
|
||||||
|
if why == "" {
|
||||||
delete(g.left, module+"/"+key)
|
delete(g.left, module+"/"+key)
|
||||||
return g.forward(module, key, value, false)
|
return g.forward(module, key, value, false)
|
||||||
}
|
}
|
||||||
if !g.left[module+"/"+key] {
|
if g.left[module+"/"+key] != why {
|
||||||
g.left[module+"/"+key] = true
|
g.left[module+"/"+key] = why
|
||||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||||
delivery.ToolchainsBucket(module), key, module, node)
|
|
||||||
}
|
}
|
||||||
return g.forward(module, key, nil, true)
|
return g.forward(module, key, nil, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||||
func openAssignment(value []byte) bool {
|
// The caller holds mu.
|
||||||
|
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||||
|
since, open := g.standing[node+"."+module]
|
||||||
|
if !open {
|
||||||
|
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||||
|
}
|
||||||
|
tc, err := delivery.DecodeToolchain(value)
|
||||||
|
if err == nil && tc.At.Before(since) {
|
||||||
|
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||||
|
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||||
|
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||||
|
// it was assigned, zero where the value does not say.
|
||||||
|
func openAssignment(value []byte) (time.Time, bool) {
|
||||||
var a struct {
|
var a struct {
|
||||||
Ended time.Time `json:"ended"`
|
Assigned time.Time `json:"assigned"`
|
||||||
State string `json:"state"`
|
Ended time.Time `json:"ended"`
|
||||||
|
State string `json:"state"`
|
||||||
}
|
}
|
||||||
if json.Unmarshal(value, &a) != nil {
|
if json.Unmarshal(value, &a) != nil {
|
||||||
return false
|
return time.Time{}, false
|
||||||
}
|
}
|
||||||
return a.Ended.IsZero() && a.State == ""
|
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||||
@@ -155,7 +190,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
|||||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||||
return errs, nil
|
return errs, nil
|
||||||
}
|
}
|
||||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||||
w, err := settle(ctx, kv, each)
|
w, err := settle(ctx, kv, each)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||||
@@ -165,7 +200,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
|||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||||
changed(ctx)
|
changed(ctx)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -44,10 +45,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||||
}
|
}
|
||||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||||
}
|
}
|
||||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||||
if !gate.Settled() {
|
if !gate.Settled() {
|
||||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||||
}
|
}
|
||||||
@@ -63,20 +64,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
t.Fatal("an unassigned node's report changed what was passed on")
|
t.Fatal("an unassigned node's report changed what was passed on")
|
||||||
}
|
}
|
||||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||||
t.Fatal("assigning ace again passed nothing on")
|
t.Fatal("assigning ace again passed nothing on")
|
||||||
}
|
}
|
||||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||||
}
|
}
|
||||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||||
t.Fatal("a pending assignment is no assignment on the machine")
|
t.Fatal("a pending assignment is no assignment on the machine")
|
||||||
}
|
}
|
||||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||||
t.Fatalf("got %q, %q", v, none)
|
t.Fatalf("got %q, %q", v, none)
|
||||||
}
|
}
|
||||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||||
t.Fatalf("got %q", v)
|
t.Fatalf("got %q", v)
|
||||||
}
|
}
|
||||||
@@ -90,23 +91,75 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||||
|
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||||
|
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||||
|
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||||
set := &buildrun.ToolchainSet{}
|
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||||
gate.Settled()
|
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
key, _ := gateReport(t, node, version)
|
||||||
t.Fatal("an assignment with no report passed something on")
|
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return key, value
|
||||||
}
|
}
|
||||||
key, value := gateReport(t, "g14", "v26.12.1")
|
for _, c := range []struct {
|
||||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
name string
|
||||||
t.Fatal("an assigned node's report was not passed on")
|
assignment string // ace's assignment value
|
||||||
}
|
revision time.Time // when the bus wrote it
|
||||||
key, value = gateReport(t, "ace", "v26.12.0")
|
at time.Time // when ace wrote its report
|
||||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
counts bool
|
||||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
}{
|
||||||
}
|
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||||
t.Fatalf("got %q, %q", v, none)
|
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||||
|
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
set := &buildrun.ToolchainSet{}
|
||||||
|
var said []string
|
||||||
|
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||||
|
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||||
|
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||||
|
key, value := gateReport(t, node, "v26.12.1")
|
||||||
|
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||||
|
}
|
||||||
|
gate.Settled()
|
||||||
|
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||||
|
t.Fatal("an assignment with no report passed something on")
|
||||||
|
}
|
||||||
|
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||||
|
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||||
|
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||||
|
}
|
||||||
|
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||||
|
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||||
|
t.Fatal("the same assignment written again changed what was passed on")
|
||||||
|
}
|
||||||
|
v, none := agreedOf(set)
|
||||||
|
if c.counts {
|
||||||
|
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||||
|
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if v != "v26.12.1" || none != "" {
|
||||||
|
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||||
|
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||||
|
}
|
||||||
|
// ace writes its report again since it was assigned: it counts.
|
||||||
|
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||||
|
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||||
|
t.Fatal("ace's new report was not passed on")
|
||||||
|
}
|
||||||
|
if n := len(set.Entries("typescript")); n != 4 {
|
||||||
|
t.Fatalf("four builders report; got %d", n)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -531,23 +531,98 @@ const MaxWords = 4096
|
|||||||
|
|
||||||
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
|
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
|
||||||
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
|
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
|
||||||
// status line, which is cut short, names it.
|
// status line, which is cut short, names it. **A failure's words are never "PASS" or "ok"** (issue 1901, its third
|
||||||
|
// meeting: a package that failed after its tests passed was worded "PASS"): go test's summary lines are dropped, and with
|
||||||
|
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
|
||||||
|
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
|
||||||
func failureWords(f Failure) string {
|
func failureWords(f Failure) string {
|
||||||
out := strings.TrimSpace(f.Output)
|
out, passed := dropSummary(strings.TrimSpace(f.Output))
|
||||||
if len(out) > MaxWords {
|
cause := strings.TrimSpace(f.Cause)
|
||||||
out = out[len(out)-MaxWords:]
|
if cause == "" {
|
||||||
// Start on a whole line, and a whole character.
|
cause = causeIn(out)
|
||||||
if i := strings.IndexByte(out, '\n'); i >= 0 {
|
|
||||||
out = "... " + strings.TrimLeft(out[i:], "\n")
|
|
||||||
}
|
|
||||||
out = strings.ToValidUTF8(out, "")
|
|
||||||
}
|
}
|
||||||
if c := strings.TrimSpace(f.Cause); c != "" && !strings.HasPrefix(firstLine(out), c) {
|
switch {
|
||||||
out = c + "\n" + out
|
case cause == "" && out == "" && passed:
|
||||||
|
cause = "the tests passed, then the test program exited with a failure: an os.Exit or a panic in TestMain or after the tests"
|
||||||
|
case cause == "" && out == "":
|
||||||
|
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as its "FAIL" line alone.
|
||||||
|
cause = "the package failed with no test to name and no panic, signal or build error in its output (an older build module sent no cause)"
|
||||||
|
}
|
||||||
|
if len(cause) > MaxCause {
|
||||||
|
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
|
||||||
|
}
|
||||||
|
// The words keep MaxWords exactly: the cause and its newline, the marker and the kept lines (review round 1).
|
||||||
|
budget := MaxWords
|
||||||
|
if cause != "" {
|
||||||
|
budget -= len(cause) + 1
|
||||||
|
}
|
||||||
|
if len(out) > budget {
|
||||||
|
const marker = "... "
|
||||||
|
tail := out[len(out)-(budget-len(marker)):]
|
||||||
|
// Start on a whole line, and a whole character.
|
||||||
|
if i := strings.IndexByte(tail, '\n'); i >= 0 {
|
||||||
|
tail = strings.TrimLeft(tail[i:], "\n")
|
||||||
|
}
|
||||||
|
out = marker + strings.ToValidUTF8(tail, "")
|
||||||
|
}
|
||||||
|
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
|
||||||
|
out = strings.TrimRight(cause+"\n"+out, "\n")
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
|
||||||
|
// push the words past MaxWords).
|
||||||
|
const MaxCause = 1024
|
||||||
|
|
||||||
|
// dropSummary drops go test's summary lines from a failure's output ("PASS", "FAIL", "ok <pkg>", "FAIL <pkg>"), and says
|
||||||
|
// whether a "PASS" was among them: the package's tests passed before it failed.
|
||||||
|
func dropSummary(out string) (string, bool) {
|
||||||
|
passed := false
|
||||||
|
lines := strings.Split(out, "\n")
|
||||||
|
kept := lines[:0]
|
||||||
|
for _, l := range lines {
|
||||||
|
// go's exact forms only (review round 1): "PASS" or "FAIL" alone, "ok \t<pkg>\t<time>", "FAIL\t<pkg>\t<time>".
|
||||||
|
switch t := strings.TrimRight(l, "\r"); {
|
||||||
|
case t == "PASS":
|
||||||
|
passed = true
|
||||||
|
case t == "FAIL", strings.HasPrefix(t, "ok \t"), strings.HasPrefix(t, "FAIL\t"):
|
||||||
|
default:
|
||||||
|
kept = append(kept, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(strings.Join(kept, "\n")), passed
|
||||||
|
}
|
||||||
|
|
||||||
|
// causeIn is a race the race detector reported, as its heading, its access and the frame that made it ("WARNING: DATA
|
||||||
|
// RACE: Write at 0x... by goroutine 9 in pkg.F.func1()"); else the first "panic:", "fatal error:" or "signal:" line of a
|
||||||
|
// failure's output, in that order; or "". **Only lines at column 0**, where go prints the runtime's: a test's indented log
|
||||||
|
// line holding the same words is not the cause (review round 1).
|
||||||
|
func causeIn(out string) string {
|
||||||
|
lines := strings.Split(out, "\n")
|
||||||
|
for i, l := range lines {
|
||||||
|
if strings.TrimRight(l, "\r") != "WARNING: DATA RACE" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
words := "WARNING: DATA RACE"
|
||||||
|
if i+1 < len(lines) && strings.TrimSpace(lines[i+1]) != "" {
|
||||||
|
words += ": " + strings.TrimSuffix(strings.TrimSpace(lines[i+1]), ":")
|
||||||
|
if i+2 < len(lines) && strings.HasPrefix(lines[i+2], " ") {
|
||||||
|
words += " in " + strings.TrimSpace(lines[i+2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return words
|
||||||
|
}
|
||||||
|
for _, prefix := range []string{"panic: ", "fatal error: ", "signal: "} {
|
||||||
|
for _, l := range lines {
|
||||||
|
if l = strings.TrimRight(l, "\r"); strings.HasPrefix(l, prefix) {
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
// firstLine is the first non-empty line of a failure's words.
|
// firstLine is the first non-empty line of a failure's words.
|
||||||
func firstLine(s string) string {
|
func firstLine(s string) string {
|
||||||
for _, l := range strings.Split(s, "\n") {
|
for _, l := range strings.Split(s, "\n") {
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the build module before this fix sent for two packages that failed after their tests passed (issue 1901, its third
|
||||||
|
// meeting: a try of mesh-controller kept "PASS" as its failure's words). Each is that build module's Output for the real
|
||||||
|
// `go test -json` of a throwaway package (mesh-catalog modules/mesh-build-go testdata/captured/go-test-json-pkg-*.jsonl):
|
||||||
|
// TestMain panicking in a goroutine after m.Run, TestMain exiting 2 after m.Run, and TestMain racing after m.Run.
|
||||||
|
const (
|
||||||
|
oldPassThenPanic = "PASS\npanic: the bus closed after the tests: nats: connection closed\ngoroutine 20 [running]:\n" +
|
||||||
|
"example.com/passthenfail.TestMain.func1()\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
|
||||||
|
"created by example.com/passthenfail.TestMain in goroutine 1\n\t/tmp/fix-1901b/passthenfail/p_test.go:13 +0x25\n" +
|
||||||
|
"FAIL\texample.com/passthenfail\t0.005s"
|
||||||
|
oldPassThenExit = "PASS\nFAIL\texample.com/mainexit\t0.004s"
|
||||||
|
// TestMain racing after m.Run passed, under go test -race (go-test-json-pkg-racemain.jsonl).
|
||||||
|
oldPassThenRace = "PASS\n==================\nWARNING: DATA RACE\nWrite at 0x000000836428 by goroutine 9:\n example.com/racemain.TestMain.func1()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x24\nPrevious write at 0x000000836428 by main goroutine:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:16 +0x44\n main.main()\n _testmain.go:48 +0x171\nGoroutine 9 (running) created at:\n example.com/racemain.TestMain()\n /tmp/fix-1901b/racemain/p_test.go:15 +0x37\n main.main()\n _testmain.go:48 +0x171\n==================\nFound 1 data race(s)\nFAIL\texample.com/racemain\t1.031s"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A failure with no cause told apart never has "PASS" or "ok" as its words: the checker takes the panic line from the
|
||||||
|
// output, or says that the tests passed and the program failed after them (1901/AC5).
|
||||||
|
func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
|
||||||
|
cases := []struct {
|
||||||
|
name, output, want string
|
||||||
|
}{
|
||||||
|
{"panic after the tests", oldPassThenPanic, "panic: the bus closed after the tests: nats: connection closed"},
|
||||||
|
{"exit after the tests", oldPassThenExit, "the tests passed, then the test program exited with a failure"},
|
||||||
|
{"race after the tests", oldPassThenRace, "WARNING: DATA RACE: Write at 0x000000836428 by goroutine 9 in example.com/racemain.TestMain.func1()"},
|
||||||
|
{"a panic deep in a long output", "PASS\npanic: the seat row vanished\n" + strings.Repeat("goroutine 7 [chan receive]:\n\tsome/frame.go:1 +0x1\n", 300) + "FAIL\tp\t2.0s", "panic: the seat row vanished"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "github.com/novox/mesh-controller/cmd/mesh-controller", Output: c.output}}})
|
||||||
|
for _, l := range strings.Split(v.Failed.Words, "\n") {
|
||||||
|
if l = strings.TrimSpace(l); l == "PASS" || strings.HasPrefix(l, "ok ") || strings.HasPrefix(l, "ok\t") {
|
||||||
|
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(v.Failed.Words, c.want) {
|
||||||
|
t.Errorf("1901/AC5: words start %q, want %q", firstLine(v.Failed.Words), c.want)
|
||||||
|
}
|
||||||
|
if got := firstFailed(*v); !strings.Contains(got, c.want) || strings.HasSuffix(got, ": PASS") {
|
||||||
|
t.Errorf("1901/AC5: status = %q", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A cause the build module told apart is cut to a line's bound, so the words keep their bound (the 1901 review's note).
|
||||||
|
func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
|
||||||
|
f := Failure{Package: "p", Test: "TestX", Cause: strings.Repeat("c", 5000), Output: "x_test.go:1: something"}
|
||||||
|
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||||
|
if len(v.Failed.Words) > MaxWords+200 {
|
||||||
|
t.Errorf("1901/AC5: words are %d bytes, bound %d", len(v.Failed.Words), MaxWords)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The build module on main sends a package that stopped (a timeout, an exit, a kill) as only go test's last line, with no
|
||||||
|
// cause: the words then say so, never nothing (review round 1). Each output is what that build module sends for the
|
||||||
|
// capture of the same name (go-test-json-pkg-timeout, -exiting, -killed): the package's own output, its "FAIL" line.
|
||||||
|
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
|
||||||
|
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
|
||||||
|
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
|
||||||
|
if !strings.HasPrefix(v.Failed.Words, "the package failed with no test to name and no panic, signal or build error in its output") {
|
||||||
|
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A line of the test's own that starts as go's summary does, or that holds a runtime's words indented, is not taken for
|
||||||
|
// either (review round 1): go prints its summary in exact forms and the runtime's lines at column 0.
|
||||||
|
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
|
||||||
|
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
|
||||||
|
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
|
||||||
|
if !strings.HasPrefix(v.Failed.Words, "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
|
||||||
|
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The words keep MaxWords exactly, counting the cause, the marker and the newlines (review round 1).
|
||||||
|
func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
|
||||||
|
for _, cause := range []string{"", "a cause", strings.Repeat("c", 3000)} {
|
||||||
|
for _, n := range []int{100, 170, 171, 172, 500} {
|
||||||
|
f := Failure{Package: "p", Test: "TestX", Cause: cause, Output: strings.Repeat("a log line of twenty\n", n) + "the end"}
|
||||||
|
if w := failureWords(f); len(w) > MaxWords {
|
||||||
|
t.Errorf("1901/AC5: cause %d bytes, %d lines: words are %d bytes, bound %d", len(cause), n, len(w), MaxWords)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,8 +7,10 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"maps"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -438,8 +440,9 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
|
|||||||
|
|
||||||
// TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual: two merged messages of mesh-sdk (its module.json at 9cb4d59,
|
// TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual: two merged messages of mesh-sdk (its module.json at 9cb4d59,
|
||||||
// whose lib names its paths) differ in the root tree, as a change to go/ alone makes them, and carry the same objects of the
|
// whose lib names its paths) differ in the root tree, as a change to go/ alone makes them, and carry the same objects of the
|
||||||
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, and its new form
|
// named paths; the module's kept tree, and so its fingerprint's tree input, is the same after both, its new form
|
||||||
// (fingerprint or reason) does not move (issue 1758, I9, ADR 0499 check clause 6 in miniature).
|
// (fingerprint or reason) does not move, and so the second writes no value at all (issue 1758, I9, ADR 0499 check clause 6
|
||||||
|
// in miniature; the operator, 2026-10-25: a module's value moves only with the module).
|
||||||
func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
||||||
manifest, err := os.ReadFile("../../internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json")
|
manifest, err := os.ReadFile("../../internal/index/testdata/sdk-9cb4d59-mesh-sdk.module.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -448,7 +451,7 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
|||||||
objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222",
|
objects := map[string]string{"src": "1111111111111111111111111111111111111111", "test": "2222222222222222222222222222222222222222",
|
||||||
"package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}
|
"package.json": "3333333333333333333333333333333333333333", "package-lock.json": "4444444444444444444444444444444444444444", "tsconfig.json": "5555555555555555555555555555555555555555"}
|
||||||
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {})
|
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {})
|
||||||
merged := func(merge, root string) delivery.Module {
|
publish := func(merge, root string, objects map[string]string) {
|
||||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70,
|
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 70,
|
||||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root},
|
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: root, Trees: map[string]string{"": root},
|
||||||
Manifests: map[string]string{"": string(manifest)}, Paths: map[string]map[string]string{"": objects}})
|
Manifests: map[string]string{"": string(manifest)}, Paths: map[string]map[string]string{"": objects}})
|
||||||
@@ -458,6 +461,8 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
|||||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.merged", data); err != nil {
|
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-sdk.merged", data); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
at := func(merge string) delivery.Module {
|
||||||
var m delivery.Module
|
var m delivery.Module
|
||||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
if e, err := modules.Get(ctx, "mesh-sdk"); err == nil {
|
if e, err := modules.Get(ctx, "mesh-sdk"); err == nil {
|
||||||
@@ -469,13 +474,35 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
|||||||
t.Fatalf("modules.mesh-sdk = %+v within 5 s, want the merge %s", m, merge)
|
t.Fatalf("modules.mesh-sdk = %+v within 5 s, want the merge %s", m, merge)
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
first := merged("89abcdef0123456789abcdef0123456789abcdef", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
|
const firstMerge, secondMerge, thirdMerge = "89abcdef0123456789abcdef0123456789abcdef", "99abcdef0123456789abcdef0123456789abcdef", "a9abcdef0123456789abcdef0123456789abcdef"
|
||||||
second := merged("99abcdef0123456789abcdef0123456789abcdef", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
|
publish(firstMerge, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", objects)
|
||||||
if first.Inputs2 == nil || second.Inputs2 == nil || first.Inputs2.Tree != second.Inputs2.Tree || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
|
first := at(firstMerge)
|
||||||
t.Fatalf("kept trees %+v and %+v, want one named tree", first.Inputs2, second.Inputs2)
|
if first.Inputs2 == nil || !strings.HasPrefix(first.Inputs2.Tree, delivery.NamedTreePrefix) {
|
||||||
|
t.Fatalf("kept inputs %+v, want a named tree", first.Inputs2)
|
||||||
}
|
}
|
||||||
if first.FP2 != second.FP2 || first.None2 != second.None2 {
|
// The go/-only merge, then one that changes src/: the stream is taken in order, so once the third is written the second
|
||||||
t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2)
|
// was heard, and it wrote nothing: the module's value stays at the commit its fingerprint first appeared at (the
|
||||||
|
// operator, 2026-10-25).
|
||||||
|
publish(secondMerge, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", objects)
|
||||||
|
changed := maps.Clone(objects)
|
||||||
|
changed["src"] = "6666666666666666666666666666666666666666"
|
||||||
|
publish(thirdMerge, "cccccccccccccccccccccccccccccccccccccccc", changed)
|
||||||
|
at(thirdMerge)
|
||||||
|
history, err := modules.History(ctx, "mesh-sdk")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var commits []string
|
||||||
|
for _, e := range history {
|
||||||
|
if m, err := delivery.DecodeModule(e.Value()); err == nil {
|
||||||
|
commits = append(commits, m.Commit)
|
||||||
|
if m.Commit == firstMerge && (m.Inputs2 == nil || m.Inputs2.Tree != first.Inputs2.Tree || m.FP2 != first.FP2 || m.None2 != first.None2) {
|
||||||
|
t.Errorf("the value at the first merge moved: %+v, was %+v", m, first)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if slices.Contains(commits, secondMerge) {
|
||||||
|
t.Errorf("the go/-only merge wrote a value: the commits written are %v", commits)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -615,17 +642,33 @@ func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
|
|||||||
return got
|
return got
|
||||||
}
|
}
|
||||||
agreed := func(m delivery.Module) bool { return m.FP2 != "" && m.Toolchains["go"].Version == "go1.27.3" }
|
agreed := func(m delivery.Module) bool { return m.FP2 != "" && m.Toolchains["go"].Version == "go1.27.3" }
|
||||||
if m := read(agreed); !agreed(m) {
|
held := read(agreed)
|
||||||
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", m.FP2, m.None2, m.Toolchains)
|
if !agreed(held) {
|
||||||
|
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains)
|
||||||
}
|
}
|
||||||
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil {
|
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
// n2's report, written since its assignment, counts: the builders disagree. A fingerprint already computed is kept
|
||||||
|
// (issue 1940), and only a new one needs the agreement: a merge moving avahi gets none until they agree.
|
||||||
|
time.Sleep(500 * time.Millisecond)
|
||||||
|
if e, err := modules.Get(ctx, "avahi"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
} else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 {
|
||||||
|
t.Fatalf("after n2's assignment avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
|
||||||
|
}
|
||||||
|
moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1)
|
||||||
|
if moved == string(data) {
|
||||||
|
t.Fatal("the merge names no tree to move")
|
||||||
|
}
|
||||||
|
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", []byte(moved)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
disagree := func(m delivery.Module) bool {
|
disagree := func(m delivery.Module) bool {
|
||||||
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
|
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
|
||||||
}
|
}
|
||||||
if m := read(disagree); !disagree(m) {
|
if m := read(disagree); !disagree(m) {
|
||||||
t.Fatalf("within 5 s of n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
|
t.Fatalf("within 5 s of a merge after n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
|
||||||
}
|
}
|
||||||
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil {
|
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -28,8 +28,11 @@ import (
|
|||||||
//
|
//
|
||||||
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||||
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||||
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
|
||||||
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
|
||||||
|
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
|
||||||
|
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
|
||||||
|
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
|
||||||
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||||
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||||
type toolchainGate struct {
|
type toolchainGate struct {
|
||||||
@@ -38,14 +41,14 @@ type toolchainGate struct {
|
|||||||
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||||
logf func(format string, a ...any)
|
logf func(format string, a ...any)
|
||||||
known bool // the assignments are settled
|
known bool // the assignments are settled
|
||||||
standing map[string]bool // <node>.<module> with an open assignment
|
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
|
||||||
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||||
left map[string]bool // <module>/<key> said as left out, so it is said once
|
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
|
||||||
}
|
}
|
||||||
|
|
||||||
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||||
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
|
||||||
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
raw: map[string]map[string][]byte{}, left: map[string]string{}}
|
||||||
for _, m := range modules {
|
for _, m := range modules {
|
||||||
g.modules[m] = true
|
g.modules[m] = true
|
||||||
}
|
}
|
||||||
@@ -67,20 +70,31 @@ func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool
|
|||||||
return g.pass(module, key)
|
return g.pass(module, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
|
||||||
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
// whether what it passed on changed.
|
||||||
|
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
|
||||||
node, module, ok := strings.Cut(key, ".")
|
node, module, ok := strings.Cut(key, ".")
|
||||||
if !ok || !g.modules[module] {
|
if !ok || !g.modules[module] {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
open := !deleted && openAssignment(value)
|
assigned, open := openAssignment(value)
|
||||||
|
open = open && !deleted
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
if g.standing[key] == open {
|
was, stood := g.standing[key]
|
||||||
|
since := assigned
|
||||||
|
if since.IsZero() {
|
||||||
|
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
|
||||||
|
since = at
|
||||||
|
if stood {
|
||||||
|
since = was
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if open == stood && (!open || since.Equal(was)) {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if open {
|
if open {
|
||||||
g.standing[key] = true
|
g.standing[key] = since
|
||||||
} else {
|
} else {
|
||||||
delete(g.standing, key)
|
delete(g.standing, key)
|
||||||
}
|
}
|
||||||
@@ -111,37 +125,58 @@ func (g *toolchainGate) Settled() bool {
|
|||||||
return changed
|
return changed
|
||||||
}
|
}
|
||||||
|
|
||||||
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
|
||||||
// otherwise. The caller holds mu.
|
// known), deleted otherwise. The caller holds mu.
|
||||||
func (g *toolchainGate) pass(module, key string) bool {
|
func (g *toolchainGate) pass(module, key string) bool {
|
||||||
value, held := g.raw[module][key]
|
value, held := g.raw[module][key]
|
||||||
if !held {
|
if !held {
|
||||||
return g.forward(module, key, nil, true)
|
return g.forward(module, key, nil, true)
|
||||||
}
|
}
|
||||||
node, _, err := delivery.ParseToolchainKey(key)
|
node, _, err := delivery.ParseToolchainKey(key)
|
||||||
if err != nil || !g.known || g.standing[node+"."+module] {
|
why := ""
|
||||||
|
if err == nil && g.known {
|
||||||
|
why = g.leftOut(module, node, value)
|
||||||
|
}
|
||||||
|
if why == "" {
|
||||||
delete(g.left, module+"/"+key)
|
delete(g.left, module+"/"+key)
|
||||||
return g.forward(module, key, value, false)
|
return g.forward(module, key, value, false)
|
||||||
}
|
}
|
||||||
if !g.left[module+"/"+key] {
|
if g.left[module+"/"+key] != why {
|
||||||
g.left[module+"/"+key] = true
|
g.left[module+"/"+key] = why
|
||||||
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
|
||||||
delivery.ToolchainsBucket(module), key, module, node)
|
|
||||||
}
|
}
|
||||||
return g.forward(module, key, nil, true)
|
return g.forward(module, key, nil, true)
|
||||||
}
|
}
|
||||||
|
|
||||||
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
|
||||||
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
|
||||||
func openAssignment(value []byte) bool {
|
// The caller holds mu.
|
||||||
|
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
|
||||||
|
since, open := g.standing[node+"."+module]
|
||||||
|
if !open {
|
||||||
|
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
|
||||||
|
}
|
||||||
|
tc, err := delivery.DecodeToolchain(value)
|
||||||
|
if err == nil && tc.At.Before(since) {
|
||||||
|
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
|
||||||
|
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
|
||||||
|
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
|
||||||
|
// it was assigned, zero where the value does not say.
|
||||||
|
func openAssignment(value []byte) (time.Time, bool) {
|
||||||
var a struct {
|
var a struct {
|
||||||
Ended time.Time `json:"ended"`
|
Assigned time.Time `json:"assigned"`
|
||||||
State string `json:"state"`
|
Ended time.Time `json:"ended"`
|
||||||
|
State string `json:"state"`
|
||||||
}
|
}
|
||||||
if json.Unmarshal(value, &a) != nil {
|
if json.Unmarshal(value, &a) != nil {
|
||||||
return false
|
return time.Time{}, false
|
||||||
}
|
}
|
||||||
return a.Ended.IsZero() && a.State == ""
|
return a.Assigned, a.Ended.IsZero() && a.State == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||||
@@ -155,7 +190,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
|||||||
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||||
return errs, nil
|
return errs, nil
|
||||||
}
|
}
|
||||||
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
|
||||||
w, err := settle(ctx, kv, each)
|
w, err := settle(ctx, kv, each)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||||
@@ -165,7 +200,7 @@ func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate,
|
|||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||||
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
|
||||||
changed(ctx)
|
changed(ctx)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -44,10 +45,10 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||||
}
|
}
|
||||||
for _, node := range []string{"g14", "novox", "shanks"} {
|
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||||
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
|
||||||
}
|
}
|
||||||
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
|
||||||
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
|
||||||
if !gate.Settled() {
|
if !gate.Settled() {
|
||||||
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||||
}
|
}
|
||||||
@@ -63,20 +64,20 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
t.Fatal("an unassigned node's report changed what was passed on")
|
t.Fatal("an unassigned node's report changed what was passed on")
|
||||||
}
|
}
|
||||||
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
|
||||||
t.Fatal("assigning ace again passed nothing on")
|
t.Fatal("assigning ace again passed nothing on")
|
||||||
}
|
}
|
||||||
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||||
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||||
}
|
}
|
||||||
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||||
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
|
||||||
t.Fatal("a pending assignment is no assignment on the machine")
|
t.Fatal("a pending assignment is no assignment on the machine")
|
||||||
}
|
}
|
||||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||||
t.Fatalf("got %q, %q", v, none)
|
t.Fatalf("got %q, %q", v, none)
|
||||||
}
|
}
|
||||||
gate.Assignment("ace.mesh-build-node", nil, true)
|
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
|
||||||
if v, _ := agreedOf(set); v != "v26.12.1" {
|
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||||
t.Fatalf("got %q", v)
|
t.Fatalf("got %q", v)
|
||||||
}
|
}
|
||||||
@@ -90,23 +91,75 @@ func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
|
||||||
|
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
|
||||||
|
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
|
||||||
|
// writes it again. An assignment whose value names no time stands since the revision that opened it.
|
||||||
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||||
set := &buildrun.ToolchainSet{}
|
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
||||||
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
|
||||||
gate.Settled()
|
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
||||||
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
key, _ := gateReport(t, node, version)
|
||||||
t.Fatal("an assignment with no report passed something on")
|
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return key, value
|
||||||
}
|
}
|
||||||
key, value := gateReport(t, "g14", "v26.12.1")
|
for _, c := range []struct {
|
||||||
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
name string
|
||||||
t.Fatal("an assigned node's report was not passed on")
|
assignment string // ace's assignment value
|
||||||
}
|
revision time.Time // when the bus wrote it
|
||||||
key, value = gateReport(t, "ace", "v26.12.0")
|
at time.Time // when ace wrote its report
|
||||||
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
counts bool
|
||||||
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
}{
|
||||||
}
|
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
|
||||||
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
|
||||||
t.Fatalf("got %q, %q", v, none)
|
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
|
||||||
|
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
set := &buildrun.ToolchainSet{}
|
||||||
|
var said []string
|
||||||
|
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
|
||||||
|
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||||
|
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
|
||||||
|
key, value := gateReport(t, node, "v26.12.1")
|
||||||
|
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||||
|
}
|
||||||
|
gate.Settled()
|
||||||
|
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
|
||||||
|
t.Fatal("an assignment with no report passed something on")
|
||||||
|
}
|
||||||
|
key, value := report(t, "ace", "v26.12.0", c.at)
|
||||||
|
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
|
||||||
|
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
||||||
|
}
|
||||||
|
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
|
||||||
|
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
|
||||||
|
t.Fatal("the same assignment written again changed what was passed on")
|
||||||
|
}
|
||||||
|
v, none := agreedOf(set)
|
||||||
|
if c.counts {
|
||||||
|
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||||
|
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if v != "v26.12.1" || none != "" {
|
||||||
|
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
|
||||||
|
t.Fatalf("said %q, want once why ace's report is left out", said)
|
||||||
|
}
|
||||||
|
// ace writes its report again since it was assigned: it counts.
|
||||||
|
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
|
||||||
|
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||||
|
t.Fatal("ace's new report was not passed on")
|
||||||
|
}
|
||||||
|
if n := len(set.Entries("typescript")); n != 4 {
|
||||||
|
t.Fatalf("four builders report; got %d", n)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,7 +22,8 @@ import (
|
|||||||
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
|
// form (Module.FP, None) is no longer written: it was computed from those records (ADR 0453 clause 6). A module the
|
||||||
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
|
// bucket holds of another repository is left as it is and said: a move between repositories is the operator's (ADR 0420
|
||||||
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
|
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
|
||||||
// A value the module already holds is not written again.
|
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
|
||||||
|
// appeared at (writeOverlay), so a merge writes only the modules it changes.
|
||||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||||
digests, err := ix.keepManifests(ctx, f)
|
digests, err := ix.keepManifests(ctx, f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -85,14 +85,51 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
|
|||||||
|
|
||||||
const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||||
|
|
||||||
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing)
|
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing); the operator 2026-10-25 (a module's value moves only with the module)
|
||||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
const repo, next = "novox/mesh-delivery", "cccccccccccccccccccccccccccccccccccccccc"
|
||||||
m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})
|
three := map[string]string{"modules/a": "treeA", "modules/b": "treeB", "modules/c": "treeC"}
|
||||||
_ = ix.Apply(context.Background(), m)
|
for _, tc := range []struct {
|
||||||
rev, writes := mods.revs["a"], mods.writes
|
name string
|
||||||
_ = ix.Apply(context.Background(), m)
|
then func(ix *Indexer) error // after the first merge of a, b and c at merge, stamped by source revision 7
|
||||||
if mods.revs["a"] != rev || mods.writes != writes {
|
// writes is the values written by then; commit is each module's commit after.
|
||||||
t.Errorf("the same merge wrote again (%d writes, was %d)", mods.writes, writes)
|
writes int
|
||||||
|
commit map[string]string
|
||||||
|
}{
|
||||||
|
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
|
||||||
|
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
|
||||||
|
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge},
|
||||||
|
then: func(ix *Indexer) error {
|
||||||
|
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
|
||||||
|
m.Merge = next
|
||||||
|
return ix.Apply(context.Background(), m)
|
||||||
|
}},
|
||||||
|
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge},
|
||||||
|
then: func(ix *Indexer) error {
|
||||||
|
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
ix, mods := modulesIndexer(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
if err := ix.Apply(ctx, mergedMsg(repo, three)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := ix.ApplySource(ctx, 7, sourceOf(repo, merge)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writes := mods.writes
|
||||||
|
if err := tc.then(ix); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got := mods.writes - writes; got != tc.writes {
|
||||||
|
t.Errorf("%d values written, want %d", got, tc.writes)
|
||||||
|
}
|
||||||
|
for module, want := range tc.commit {
|
||||||
|
if m, ok := moduleOf(t, mods, module); !ok || m.Commit != want {
|
||||||
|
t.Errorf("%s = %+v (%v), want commit %.8s", module, m, ok, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -112,12 +112,12 @@ func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// previewOf is the preview of a head merged onto the trunk head the source names now: found, or stale when only a preview
|
// previewOf is the preview of a head merged onto the trunk head the source heads name: found, or stale when only a preview
|
||||||
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
|
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
|
||||||
func (ix *Indexer) previewOf(repository, head string) (pv delivery.Preview, found, stale bool) {
|
func (ix *Indexer) previewOf(heads map[string]string, repository, head string) (pv delivery.Preview, found, stale bool) {
|
||||||
ix.mu.Lock()
|
ix.mu.Lock()
|
||||||
defer ix.mu.Unlock()
|
defer ix.mu.Unlock()
|
||||||
trunk := ix.source.heads[repository]
|
trunk := heads[repository]
|
||||||
if trunk == "" {
|
if trunk == "" {
|
||||||
return delivery.Preview{}, false, false
|
return delivery.Preview{}, false, false
|
||||||
}
|
}
|
||||||
@@ -137,14 +137,23 @@ func (ix *Indexer) previewOf(repository, head string) (pv delivery.Preview, foun
|
|||||||
// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
|
// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
|
||||||
// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with
|
// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with
|
||||||
// no preview keeps what it had.
|
// no preview keeps what it had.
|
||||||
|
//
|
||||||
|
// The candidate's Base is the source revision whose trunk heads the previews were read onto (novox/hq ADR 0437 §5): written
|
||||||
|
// whenever the fingerprints are, so the builder merges a preview onto the trunk head its fingerprint was computed on (the head
|
||||||
|
// of the pull request's repository in that revision), never onto a module value's commit, which stays at the commit its
|
||||||
|
// fingerprint first appeared at.
|
||||||
func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool {
|
func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool {
|
||||||
changed := false
|
ix.mu.Lock()
|
||||||
|
rev, heads := ix.source.rev, ix.source.heads
|
||||||
|
ix.mu.Unlock()
|
||||||
|
changed, read := false, false
|
||||||
for i := range c.Pulls {
|
for i := range c.Pulls {
|
||||||
p := &c.Pulls[i]
|
p := &c.Pulls[i]
|
||||||
pv, found, stale := ix.previewOf(p.Repository, p.Head.Head)
|
pv, found, stale := ix.previewOf(heads, p.Repository, p.Head.Head)
|
||||||
if !found && !stale {
|
if !found && !stale {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
read = true
|
||||||
// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
|
// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
|
||||||
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
|
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
|
||||||
dirs := map[string]string{}
|
dirs := map[string]string{}
|
||||||
@@ -198,6 +207,9 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
|
|||||||
if changed {
|
if changed {
|
||||||
c.FPSet = delivery.FingerprintSet(c.Pulls)
|
c.FPSet = delivery.FingerprintSet(c.Pulls)
|
||||||
}
|
}
|
||||||
|
if read && rev != 0 && c.Base != rev && (changed || c.Base == 0) {
|
||||||
|
c.Base, changed = rev, true
|
||||||
|
}
|
||||||
return changed
|
return changed
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package index
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -87,9 +89,16 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
|
|||||||
if got.FP2 != trunk.FP2 || got.None2 != "" || got.Manifest != delivery.ManifestDigest([]byte(avahiOnTools(t))) {
|
if got.FP2 != trunk.FP2 || got.None2 != "" || got.Manifest != delivery.ManifestDigest([]byte(avahiOnTools(t))) {
|
||||||
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.FP2)
|
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.FP2)
|
||||||
}
|
}
|
||||||
if c := candidateOf(t, cands); c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
|
c := candidateOf(t, cands)
|
||||||
|
if c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
|
||||||
t.Errorf("fpset = %q", c.FPSet)
|
t.Errorf("fpset = %q", c.FPSet)
|
||||||
}
|
}
|
||||||
|
// The candidate names the source revision the preview was read onto (5, whose catalogue head is the preview's trunk):
|
||||||
|
// the builder merges the preview onto that head, never onto the module value's commit.
|
||||||
|
if c.Base != 5 {
|
||||||
|
t.Errorf("candidate base = %d, want the source revision 5 its preview was read onto", c.Base)
|
||||||
|
}
|
||||||
|
written(t, writtenCandidate{Source: sourceRev{Rev: 5, Heads: map[string]string{catalogueRepo: trunkNow}}, Candidate: c})
|
||||||
// Another tree gives another fingerprint.
|
// Another tree gives another fingerprint.
|
||||||
pv := avahiPreview(t, trunkNow, h1)
|
pv := avahiPreview(t, trunkNow, h1)
|
||||||
m := pv.Modules["modules/avahi"]
|
m := pv.Modules["modules/avahi"]
|
||||||
@@ -191,3 +200,38 @@ func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *t
|
|||||||
t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got)
|
t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writtenCandidate is what the builder's tests read of the indexer (mesh-delivery-builder internal/build): a candidate as the
|
||||||
|
// indexer writes it and the gate's source revision it names, kept in testdata/candidate-written.json so the builder is tested
|
||||||
|
// on what the indexer writes, never on a candidate made by hand. MESH_WRITE_TESTDATA=1 writes it again.
|
||||||
|
type writtenCandidate struct {
|
||||||
|
Source sourceRev `json:"source"`
|
||||||
|
Candidate delivery.Candidate `json:"candidate"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type sourceRev struct {
|
||||||
|
Rev uint64 `json:"rev"`
|
||||||
|
Heads map[string]string `json:"heads"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func written(t *testing.T, w writtenCandidate) {
|
||||||
|
t.Helper()
|
||||||
|
data, err := json.MarshalIndent(w, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data = append(data, '\n')
|
||||||
|
const file = "testdata/candidate-written.json"
|
||||||
|
if os.Getenv("MESH_WRITE_TESTDATA") == "1" {
|
||||||
|
if err := os.WriteFile(file, data, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
kept, err := os.ReadFile(file)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(kept) != string(data) {
|
||||||
|
t.Errorf("%s is not what the indexer writes now (MESH_WRITE_TESTDATA=1 writes it again):\n%s", file, data)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -32,8 +32,8 @@ func (ix *Indexer) srcOf(repository, merge string) uint64 {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each
|
// ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each
|
||||||
// repository the source names, each module built from it whose fingerprint was computed at the head the source names is
|
// repository the source names, each module built from it whose fingerprint was computed at the head the source names, and
|
||||||
// written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped
|
// that carries no Src yet, is written again with Src the revision (once: a later revision leaves it); a module computed at another commit is left (a merge heard later is stamped
|
||||||
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The
|
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing. The
|
||||||
// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6).
|
// modules of a repository are those the modules bucket holds of it (ADR 0501 clause 6).
|
||||||
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error {
|
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error {
|
||||||
@@ -71,7 +71,9 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
|
|||||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
}
|
}
|
||||||
if cur.Repository != repository || cur.Commit != head || cur.Src == rev {
|
// Stamped once, with the first revision naming the commit as the head: a later revision (a merge of another
|
||||||
|
// repository) leaves the value as it is, so no revision of the source rewrites a module that did not change.
|
||||||
|
if cur.Repository != repository || cur.Commit != head || cur.Src != 0 {
|
||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
}
|
}
|
||||||
cur.Src = rev
|
cur.Src = rev
|
||||||
|
|||||||
@@ -38,15 +38,15 @@ func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.
|
|||||||
if c, ok := moduleOf(t, mods, "c"); ok {
|
if c, ok := moduleOf(t, mods, "c"); ok {
|
||||||
t.Errorf("module c = %+v: written for a head no merge message of the indexer carried", c)
|
t.Errorf("module c = %+v: written for a head no merge message of the indexer carried", c)
|
||||||
}
|
}
|
||||||
// The same revision again changes nothing; a later revision moves Src.
|
// The same revision again changes nothing, and so does a later one: Src is the first revision naming the commit.
|
||||||
if err := ix.ApplySource(ctx, 7, s); err != nil {
|
if err := ix.ApplySource(ctx, 7, s); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := ix.ApplySource(ctx, 9, s); err != nil {
|
if err := ix.ApplySource(ctx, 9, s); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
|
if m, _ := moduleOf(t, mods, "a"); m.Src != 7 {
|
||||||
t.Errorf("module a Src = %d, want 9", m.Src)
|
t.Errorf("module a Src = %d, want 7", m.Src)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"source": {
|
||||||
|
"rev": 5,
|
||||||
|
"heads": {
|
||||||
|
"novox/mesh-catalog": "5555555555555555555555555555555555555555"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"candidate": {
|
||||||
|
"v": 1,
|
||||||
|
"group": "feat/x",
|
||||||
|
"base": 5,
|
||||||
|
"heads": "dfe9fb8f79e8",
|
||||||
|
"pulls": [
|
||||||
|
{
|
||||||
|
"repository": "novox/mesh-catalog",
|
||||||
|
"number": 301,
|
||||||
|
"head": "0123456789abcdef0123456789abcdef01234567",
|
||||||
|
"modules": [
|
||||||
|
"avahi"
|
||||||
|
],
|
||||||
|
"fps": {
|
||||||
|
"avahi": {
|
||||||
|
"fp2": "src2:e8d595aea1b13f1f5578ef1f245dab0d28a351c28079ecdcd2276c63a932d7be",
|
||||||
|
"manifest": "8a3856b6c0566d76f2556fc4f46c5251dc7935517bb6bd15809ded7e5546a0d8"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"fpset": "01c840ec276b"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -112,6 +112,9 @@ type known struct {
|
|||||||
languages map[string]string
|
languages map[string]string
|
||||||
bases []string
|
bases []string
|
||||||
contexts bool
|
contexts bool
|
||||||
|
// held is the fingerprint the bucket holds for the module and the toolchains it was computed with (for a merge, the
|
||||||
|
// value before it): what stillComputed keeps while the builders agree on no toolchain.
|
||||||
|
held delivery.Module
|
||||||
}
|
}
|
||||||
|
|
||||||
// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
|
// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
|
||||||
@@ -159,20 +162,24 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
|||||||
stale := map[string]bool{} // values an SDK that could not read their manifest wrote, judged again (issue 1923)
|
stale := map[string]bool{} // values an SDK that could not read their manifest wrote, judged again (issue 1923)
|
||||||
all := map[string]*known{}
|
all := map[string]*known{}
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
cur, ok := overlay[key]
|
cur, merged := overlay[key]
|
||||||
if !ok {
|
var before delivery.Module
|
||||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("modules.%s: %w", key, err)
|
return fmt.Errorf("modules.%s: %w", key, err)
|
||||||
}
|
}
|
||||||
if !found {
|
if found {
|
||||||
continue
|
if before, err = delivery.DecodeModule(value); err != nil && !merged {
|
||||||
}
|
|
||||||
if cur, err = delivery.DecodeModule(value); err != nil {
|
|
||||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !merged {
|
||||||
|
if !found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cur = before
|
||||||
|
}
|
||||||
held[key] = cur
|
held[key] = cur
|
||||||
if cur.Manifest == "" || cur.Inputs2 == nil {
|
if cur.Manifest == "" || cur.Inputs2 == nil {
|
||||||
continue
|
continue
|
||||||
@@ -186,7 +193,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
|||||||
cur, stale[key] = again, true
|
cur, stale[key] = again, true
|
||||||
held[key] = cur
|
held[key] = cur
|
||||||
}
|
}
|
||||||
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)}
|
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m), held: before}
|
||||||
}
|
}
|
||||||
tools := ix.agreed()
|
tools := ix.agreed()
|
||||||
type result struct {
|
type result struct {
|
||||||
@@ -251,6 +258,9 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
|||||||
r.none = strings.Join(missing, "; ")
|
r.none = strings.Join(missing, "; ")
|
||||||
case noToolchain != "":
|
case noToolchain != "":
|
||||||
r.none = noToolchain
|
r.none = noToolchain
|
||||||
|
if fp, ok := stillComputed(module, kept, k, toolchains); ok {
|
||||||
|
r = result{fp: fp, tcs: k.held.Toolchains}
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
|
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
|
||||||
if r.fp != "" {
|
if r.fp != "" {
|
||||||
@@ -309,6 +319,12 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
|
|||||||
|
|
||||||
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
|
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
|
||||||
// the ones that were computed, the others are written as the merge gave them.
|
// the ones that were computed, the others are written as the merge gave them.
|
||||||
|
//
|
||||||
|
// **A module the merge did not change is not written** (the operator, 2026-10-25: "only if a module has got a new version in
|
||||||
|
// commit x, should a new module version get created linked to commit x"). A value names the commit its fingerprint first
|
||||||
|
// appeared at: when the merge gives a module the value it holds but for the commit (and the source revision stamped on it),
|
||||||
|
// the held value stands, its commit unchanged, so a catalogue merge touching one module writes one key, not one per module of
|
||||||
|
// the repository, and the controller's take-in hears only that one. The repository's trunk head is the gate's source.main.
|
||||||
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error {
|
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error {
|
||||||
for _, key := range slices.Sorted(maps.Keys(overlay)) {
|
for _, key := range slices.Sorted(maps.Keys(overlay)) {
|
||||||
next := overlay[key]
|
next := overlay[key]
|
||||||
@@ -317,7 +333,7 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
|
|||||||
}
|
}
|
||||||
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||||
if found {
|
if found {
|
||||||
if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
|
if cur, err := delivery.DecodeModule(value); err == nil && sameButWhere(cur, next) {
|
||||||
return nil, false, nil
|
return nil, false, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -331,6 +347,13 @@ func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// sameButWhere says the held value cur is next but for where it was computed (Commit and Src): the module is unchanged.
|
||||||
|
func sameButWhere(cur, next delivery.Module) bool {
|
||||||
|
next = withV(next)
|
||||||
|
next.Commit, next.Src = cur.Commit, cur.Src
|
||||||
|
return reflect.DeepEqual(cur, next)
|
||||||
|
}
|
||||||
|
|
||||||
// manifestRaw is manifests.<digest>: the manifest as stored and parsed by the running SDK.
|
// manifestRaw is manifests.<digest>: the manifest as stored and parsed by the running SDK.
|
||||||
func (ix *Indexer) manifestRaw(ctx context.Context, digest string) (string, catalogue.Manifest, error) {
|
func (ix *Indexer) manifestRaw(ctx context.Context, digest string) (string, catalogue.Manifest, error) {
|
||||||
key, err := delivery.ManifestsKey(digest)
|
key, err := delivery.ManifestsKey(digest)
|
||||||
@@ -381,6 +404,32 @@ func hasContext(m catalogue.Manifest) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stillComputed is the fingerprint a module already holds, kept while a language it is built with has no agreed toolchain
|
||||||
|
// (issue 1940, 2026-10-25 20:56: a build node assigned again brought a stale report into the agreement, every module's
|
||||||
|
// fingerprint was blanked to "builders disagree", and the controller's take-in refused modules whose fingerprints were
|
||||||
|
// computed and built). Only computing a new fingerprint needs the agreement: the held one is kept when its kept inputs,
|
||||||
|
// with the toolchains it was computed with in place of the languages agreed on none, give it again. Any other input that
|
||||||
|
// moved (a merge, a base, an agreed toolchain) gives another fingerprint, and the module has none until they agree.
|
||||||
|
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2) (string, bool) {
|
||||||
|
if k.held.FP2 == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
toolchains := map[string]delivery.Toolchain2{}
|
||||||
|
for artifact, language := range k.languages {
|
||||||
|
if tc, ok := agreed[artifact]; ok {
|
||||||
|
toolchains[artifact] = tc
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
used, ok := k.held.Toolchains[language]
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
|
||||||
|
}
|
||||||
|
fp, none := delivery.SourceFingerprint2(kept.With(module, toolchains))
|
||||||
|
return fp, none == "" && fp == k.held.FP2
|
||||||
|
}
|
||||||
|
|
||||||
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
|
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
|
||||||
func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed {
|
func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed {
|
||||||
if len(by) == 0 {
|
if len(by) == 0 {
|
||||||
|
|||||||
@@ -123,9 +123,16 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) { // 1444/AC6
|
func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) { // 1444/AC6
|
||||||
|
// Both reports are read before the module's first fingerprint is computed (a fingerprint computed before they
|
||||||
|
// disagree is kept: TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith).
|
||||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
for node, version := range map[string]string{"novox": "go1.27.1", "ace": "go1.26.0"} {
|
||||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
|
key, _ := delivery.ToolchainKey(node, "go")
|
||||||
|
ix.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", version, "1"), false)
|
||||||
|
}
|
||||||
|
if err := ix.Recompute(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
m, _ := moduleOf(t, mods, "avahi")
|
m, _ := moduleOf(t, mods, "avahi")
|
||||||
if !strings.Contains(m.None2, "builders disagree") || !strings.Contains(m.None2, "go1.27.1") || !strings.Contains(m.None2, "go1.26.0") {
|
if !strings.Contains(m.None2, "builders disagree") || !strings.Contains(m.None2, "go1.27.1") || !strings.Contains(m.None2, "go1.26.0") {
|
||||||
t.Errorf("none2 = %q, want 'builders disagree' naming both versions", m.None2)
|
t.Errorf("none2 = %q, want 'builders disagree' naming both versions", m.None2)
|
||||||
@@ -260,11 +267,13 @@ func TestAMergeKeepsAFingerprintTheModuleStillHas(t *testing.T) { // 1444 (revie
|
|||||||
w := mods.writes
|
w := mods.writes
|
||||||
// the same catalogue merged again at a later commit: nothing moved
|
// the same catalogue merged again at a later commit: nothing moved
|
||||||
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
|
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||||
|
f.Merge = otherMerge
|
||||||
if err := ix.Apply(ctx, f); err != nil {
|
if err := ix.Apply(ctx, f); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || mods.writes != w {
|
// and the commit stays the one the fingerprint first appeared at (the operator, 2026-10-25)
|
||||||
t.Errorf("fp2 = %q (was %q), writes %d more", after.FP2, before.FP2, mods.writes-w)
|
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || after.Commit != before.Commit || mods.writes != w {
|
||||||
|
t.Errorf("fp2 = %q at %.8s (was %q at %.8s), writes %d more", after.FP2, after.Commit, before.FP2, before.Commit, mods.writes-w)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -288,9 +297,24 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
|
|||||||
if n.FP2 == m.FP2 || n.Toolchains["go"].Version != "go1.27.2" {
|
if n.FP2 == m.FP2 || n.Toolchains["go"].Version != "go1.27.2" {
|
||||||
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.FP2, m.FP2, n.Toolchains)
|
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.FP2, m.FP2, n.Toolchains)
|
||||||
}
|
}
|
||||||
// Nodes that disagree leave no fingerprint and no toolchains.
|
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
|
||||||
|
// stale report blanked every module and the controller's take-in refused modules already built); only a new
|
||||||
|
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.
|
||||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
|
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
|
||||||
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil {
|
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != n.FP2 || !reflect.DeepEqual(d.Toolchains, n.Toolchains) || d.None2 != "" {
|
||||||
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v, want neither", d.FP2, d.Toolchains)
|
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v none2 %q, want the fingerprint kept (%q) with %+v", d.FP2, d.Toolchains, d.None2, n.FP2, n.Toolchains)
|
||||||
|
}
|
||||||
|
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||||
|
f.Trees["modules/avahi"] = "tree-avahi-moved"
|
||||||
|
if err := ix.Apply(context.Background(), f); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil || !strings.HasPrefix(d.None2, "builders disagree") {
|
||||||
|
t.Errorf("a new tree while nodes disagree: fp2 %q toolchains %+v none2 %q, want no fingerprint and the disagreement", d.FP2, d.Toolchains, d.None2)
|
||||||
|
}
|
||||||
|
// They agree again: the new tree gets its fingerprint.
|
||||||
|
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.2")
|
||||||
|
if a, _ := moduleOf(t, mods, "avahi"); a.FP2 == "" || a.FP2 == n.FP2 {
|
||||||
|
t.Errorf("agreed again: fp2 %q (was %q), want a new fingerprint", a.FP2, n.FP2)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,213 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
|
||||||
|
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The assigner's asks reach the operator (issue 1906). The assigner writes its asks (the hold of a fingerprint, the restore
|
||||||
|
// of a data module, the backup before an apply and a move of a module of policy record) in its bucket asks; until now the
|
||||||
|
// teller only filled them from an answer given in the view, so an ask nobody opened the view for reached nobody: on
|
||||||
|
// 2026-10-25 seven modules stayed held on every machine and the operator was never asked. Now the teller raises each on the
|
||||||
|
// operator channel as it raises the gate's merge asks: as the asker mesh-delivery-teller, once per router id, through the same
|
||||||
|
// records, the same wait for room and the same warrants. The operator's answer, from the phone or from the view, is written
|
||||||
|
// under the ask's own key in the bucket answers, which the assigner reads (mesh-delivery-assigner pass.go answeredWith); none
|
||||||
|
// of these asks needs proof of who answered (mesh-sdk go/delivery AskNeedsProof).
|
||||||
|
|
||||||
|
// mostAssignerOpen is how many of the assigner's asks the teller keeps open at the router at once. The router keeps at most
|
||||||
|
// asks.MostOpen open asks of one asker, and the teller asks both the gate's merge asks and the assigner's as one asker, so a burst
|
||||||
|
// of holds (seven at once on 2026-10-25) would take all the room and a merge Approve could not be asked; one place is always
|
||||||
|
// left for a merge ask. An assigner ask beyond these waits in the wait for room and is asked when one of them ends.
|
||||||
|
const mostAssignerOpen = asks.MostOpen - 1
|
||||||
|
|
||||||
|
// assignerOpenRecord is the key of the router ids of the assigner's asks the teller asked.
|
||||||
|
const assignerOpenRecord = "assigner.open"
|
||||||
|
|
||||||
|
// assignerOpen is the router ids of the assigner's asks open now: those asked whose ask is neither answered nor ended. An id
|
||||||
|
// that ended is dropped from the record as it is read.
|
||||||
|
func (t *Teller) assignerOpen() ([]string, *stdio.StateEntry) {
|
||||||
|
e, err := t.Store.Get(assignerOpenRecord)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
return nil, e
|
||||||
|
}
|
||||||
|
var ids []string
|
||||||
|
if err := json.Unmarshal(e.Value, &ids); err != nil {
|
||||||
|
t.logf("the record %s cannot be read: %v", assignerOpenRecord, err)
|
||||||
|
return nil, e
|
||||||
|
}
|
||||||
|
return slices.DeleteFunc(ids, func(id string) bool {
|
||||||
|
k, _, err := t.keptOf(id)
|
||||||
|
return err == nil && !open(k)
|
||||||
|
}), e
|
||||||
|
}
|
||||||
|
|
||||||
|
// assignerRoom says whether another of the assigner's asks may be asked now.
|
||||||
|
func (t *Teller) assignerRoom() bool {
|
||||||
|
ids, _ := t.assignerOpen()
|
||||||
|
return len(ids) < mostAssignerOpen
|
||||||
|
}
|
||||||
|
|
||||||
|
// assignerAsked keeps that the assigner's ask with this router id is asked.
|
||||||
|
func (t *Teller) assignerAsked(id string) {
|
||||||
|
ids, e := t.assignerOpen()
|
||||||
|
if !slices.Contains(ids, id) {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if e == nil {
|
||||||
|
_, err = t.Store.Create(assignerOpenRecord, ids)
|
||||||
|
} else {
|
||||||
|
_, err = t.Store.Update(assignerOpenRecord, ids, e.Revision)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.logf("the record %s could not be kept: %v", assignerOpenRecord, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// waitForAssignerRoom puts an assigner ask in the wait for room, once.
|
||||||
|
func (t *Teller) waitForAssignerRoom(key, id string) {
|
||||||
|
ws, rec, err := t.waiting()
|
||||||
|
if err != nil {
|
||||||
|
t.logf("%v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if slices.ContainsFunc(ws, func(x waitingAsk) bool { return x.ID == id }) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if t.keepWaiting(append(ws, waitingAsk{Key: key, ID: id}), rec) == nil {
|
||||||
|
t.logf("%s: the assigner's ask %s waits: %d of the assigner's asks are open, the most the teller keeps open so a merge ask always has room", key, id, mostAssignerOpen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// isAssignerKey says whether an ask key is one of the assigner's, not the gate's merge ask.
|
||||||
|
func isAssignerKey(key string) bool {
|
||||||
|
k, err := delivery.ParseAnyAskKey(key)
|
||||||
|
return err == nil && k.Kind != delivery.AskKindMerge
|
||||||
|
}
|
||||||
|
|
||||||
|
// assignerAsk reads the assigner's ask at key: found false when the key holds none.
|
||||||
|
func (t *Teller) assignerAsk(key string) (delivery.AssignerAsk, bool, error) {
|
||||||
|
if t.Assigner == nil {
|
||||||
|
return delivery.AssignerAsk{}, false, fmt.Errorf("the teller reads no assigner asks")
|
||||||
|
}
|
||||||
|
raw, found, err := t.Assigner.Get(key)
|
||||||
|
if err != nil || !found {
|
||||||
|
return delivery.AssignerAsk{}, false, err
|
||||||
|
}
|
||||||
|
a, err := delivery.DecodeAssignerAsk(key, raw)
|
||||||
|
if err != nil {
|
||||||
|
return delivery.AssignerAsk{}, false, fmt.Errorf("the assigner's key %s holds no ask: %w", key, err)
|
||||||
|
}
|
||||||
|
return a, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// raiseAssignerAsk asks the operator the assigner's ask at key, once per router id: an ask answered already (from the view or
|
||||||
|
// the phone) is not asked; an earlier asking still open is taken back first; an asking already made is only caught up with
|
||||||
|
// at the router, and one that ended is not asked again, for the router drops a second ask under an id it keeps.
|
||||||
|
func (t *Teller) raiseAssignerAsk(key string) {
|
||||||
|
a, found, err := t.assignerAsk(key)
|
||||||
|
if err != nil {
|
||||||
|
t.logf("%s: the ask is not raised: %v", key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
id := a.Ask.ID
|
||||||
|
if raw, _, has, err := t.Answers.Get(delivery.AnswerKey(key)); err != nil {
|
||||||
|
t.logf("%s: the answers could not be read, so the ask is not raised: %v", key, err)
|
||||||
|
return
|
||||||
|
} else if has && answeredAsking(raw) == id {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
prior, grec, err := t.seenAt(gateRecordKey(key))
|
||||||
|
if err != nil {
|
||||||
|
t.logf("%s: %v", key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if grec != nil && prior.ID != id {
|
||||||
|
t.cancelOpen(key, prior.ID) // a new asking of the same key: the earlier one is taken back
|
||||||
|
}
|
||||||
|
k, _, err := t.keptOf(id)
|
||||||
|
if err != nil {
|
||||||
|
t.logf("%s: %v", key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if k != nil {
|
||||||
|
if grec == nil || prior.ID != id {
|
||||||
|
_ = t.note(id, key, 0, grec)
|
||||||
|
}
|
||||||
|
if open(k) {
|
||||||
|
t.catchUp(id)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := t.note(id, key, 0, grec); err != nil {
|
||||||
|
t.logf("%s: the record of the ask %s could not be kept, so it is not raised: %v", key, id, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !t.assignerRoom() {
|
||||||
|
t.waitForAssignerRoom(key, id)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := t.Asker.Ask(a.Ask); err != nil {
|
||||||
|
t.logf("%s: the ask %s was not asked: %v", key, id, err)
|
||||||
|
if kept, _, _ := t.keptOf(id); open(kept) {
|
||||||
|
_ = t.Store.Delete(keptKey(id)) // kept but not published: forgotten, so the next writing of the key asks it
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.assignerAsked(id)
|
||||||
|
t.logf("%s: the assigner's ask %s is asked of the operator (%s)", key, id, a.Ask.Headline)
|
||||||
|
}
|
||||||
|
|
||||||
|
// OnAssignerAskGone handles a key the assigner took away from its asks (a hold it lifted by itself): the open router ask of
|
||||||
|
// its latest asking is taken back, so the operator is not asked about what no longer holds.
|
||||||
|
func (t *Teller) OnAssignerAskGone(key string) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if !isAssignerKey(key) {
|
||||||
|
t.logf("the assigner's key %s was taken away and is no ask key of the assigner, and is left", key)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
prior, grec, err := t.seenAt(gateRecordKey(key))
|
||||||
|
if err != nil || grec == nil {
|
||||||
|
if err != nil {
|
||||||
|
t.logf("%s: %v", key, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.cancelOpen(key, prior.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// answerAssigner is the act of a warrant for one of the assigner's asks: the ask must still stand at its key under the asking
|
||||||
|
// and the words the operator answered, and then the answer is written under the ask's key, once. Its options bind no act:
|
||||||
|
// the assigner reads the option chosen (asker.Client.Take has checked the option, its level and the ask's digest).
|
||||||
|
func (t *Teller) answerAssigner(key string, o asks.Option, w asks.Warrant) error {
|
||||||
|
a, found, err := t.assignerAsk(key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.logf("%s: the ask %s was answered and the assigner has taken it away: the answer changes nothing", key, w.Ask)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if a.Ask.ID != w.Ask {
|
||||||
|
t.logf("%s: the ask %s was answered, and the assigner has asked again under %s: an answer to an earlier asking changes nothing", key, w.Ask, a.Ask.ID)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if a.Ask.Digest() != w.AskDigest {
|
||||||
|
return fmt.Errorf("the assigner's ask %s is no longer the ask the operator answered (its digest is %s, the warrant's %s)", key, a.Ask.Digest(), w.AskDigest)
|
||||||
|
}
|
||||||
|
body, err := delivery.EncodeAnswer(delivery.Answer{Option: o.ID, Warrant: warrantID(w), Words: w.Says()})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return t.writeAnswer(key, w.Ask, body, w.Says())
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1906/H1: given the assigner's hold ask in its bucket and no answer, when the teller hears the key, then it raises the ask on
|
||||||
|
// the operator channel as itself, unchanged and once (the key heard again, as a watch replays it, asks nothing more).
|
||||||
|
func TestAnAssignerAskIsRaisedOnTheOperatorChannelOnce(t *testing.T) {
|
||||||
|
b := newBed(t)
|
||||||
|
assigner, _ := withView(b)
|
||||||
|
key, value, ask := holdAsk(t)
|
||||||
|
assigner[key] = value
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
|
||||||
|
if len(asked) != 1 {
|
||||||
|
t.Fatalf("published %d asks, want one; log:\n%s", len(asked), b.logs.String())
|
||||||
|
}
|
||||||
|
var got asks.Ask
|
||||||
|
if err := json.Unmarshal(asked[0].body, &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got.ID != ask.ID || got.Digest() != ask.Digest() {
|
||||||
|
t.Errorf("the router's ask %+v differs from the assigner's", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/H1: without the view's bucket the assigner's asks are still raised.
|
||||||
|
func TestAnAssignerAskIsRaisedWithoutTheView(t *testing.T) {
|
||||||
|
b := newBed(t)
|
||||||
|
assigner := memGate{}
|
||||||
|
b.t.Assigner = assigner
|
||||||
|
key, value, _ := holdAsk(t)
|
||||||
|
assigner[key] = value
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 1 {
|
||||||
|
t.Fatalf("published %d asks, want one; log:\n%s", n, b.logs.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/H1: the operator's answer on the phone comes back by the same path as the view's: written under the ask's own key in
|
||||||
|
// the bucket answers, naming the asking, once; a second warrant writes nothing more and nothing is said as a fault.
|
||||||
|
func TestTheOperatorsAnswerToAnAssignerAskIsWrittenUnderItsKey(t *testing.T) {
|
||||||
|
b := newBed(t)
|
||||||
|
assigner, _ := withView(b)
|
||||||
|
key, value, ask := holdAsk(t)
|
||||||
|
assigner[key] = value
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
w := ackFor(ask, "release")
|
||||||
|
b.deliver(t, w)
|
||||||
|
raw, ok := b.answers.written[delivery.AnswerKey(key)]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("no answer under %s; log:\n%s", key, b.logs.String())
|
||||||
|
}
|
||||||
|
if a, err := delivery.DecodeAnswer(raw); err != nil || a.Option != "release" || answeredAsking(raw) != ask.ID {
|
||||||
|
t.Errorf("answer = %+v, %v", a, err)
|
||||||
|
}
|
||||||
|
b.deliver(t, w)
|
||||||
|
if b.answers.writes != 1 || len(b.conds) != 0 {
|
||||||
|
t.Errorf("%d writes, conditions %+v", b.answers.writes, b.conds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/H1: an answer to an asking the assigner has since taken away or asked again changes nothing.
|
||||||
|
func TestAnAnswerToAnAssignerAskTakenAwayWritesNothing(t *testing.T) {
|
||||||
|
b := newBed(t)
|
||||||
|
assigner, _ := withView(b)
|
||||||
|
key, value, ask := holdAsk(t)
|
||||||
|
assigner[key] = value
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
delete(assigner, key)
|
||||||
|
b.deliver(t, ackFor(ask, "release"))
|
||||||
|
if b.answers.writes != 0 {
|
||||||
|
t.Errorf("an answer to a withdrawn ask was written")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/H1, H2: an ask answered in the view is taken back from the phone, and an ask the assigner took away (a hold it lifted by
|
||||||
|
// itself) is taken back too; an ask answered before it is heard is never raised.
|
||||||
|
func TestAnAssignerAskAnsweredInTheViewOrTakenAwayIsTakenBack(t *testing.T) {
|
||||||
|
cancelled := func(b *bed, id string) bool {
|
||||||
|
for _, p := range b.askedOf(asks.CancelSubject(delivery.TellerModule)) {
|
||||||
|
if string(p.body) == `{"id":"`+id+`"}` {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
b := newBed(t)
|
||||||
|
assigner, view := withView(b)
|
||||||
|
key, value, ask := holdAsk(t)
|
||||||
|
assigner[key] = value
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
view[key+".view"] = viewAnswer(t, "release", "")
|
||||||
|
b.t.OnViewAnswer(key + ".view")
|
||||||
|
if !cancelled(b, ask.ID) {
|
||||||
|
t.Errorf("the ask answered in the view was not taken back; log:\n%s", b.logs.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
b = newBed(t)
|
||||||
|
assigner, _ = withView(b)
|
||||||
|
assigner[key] = value
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
delete(assigner, key)
|
||||||
|
b.t.OnAssignerAskGone(key)
|
||||||
|
if !cancelled(b, ask.ID) {
|
||||||
|
t.Errorf("the ask taken away was not taken back; log:\n%s", b.logs.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
b = newBed(t)
|
||||||
|
assigner, view = withView(b)
|
||||||
|
assigner[key] = value
|
||||||
|
view[key+".view"] = viewAnswer(t, "keep", "")
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 0 {
|
||||||
|
t.Errorf("an ask answered in the view was raised %d times", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/H1: an assigner ask refused for want of room waits and is asked again when another of the teller's asks ends, as a merge
|
||||||
|
// ask does; a refusal for its words is said as a condition about a delivery ask, not a merge.
|
||||||
|
func TestAnAssignerAskWaitsForRoomAndARefusalIsSaidAsADeliveryAsk(t *testing.T) {
|
||||||
|
b := newBed(t)
|
||||||
|
assigner, _ := withView(b)
|
||||||
|
_, first := b.askOf(t, 1, 1)
|
||||||
|
b.askOf(t, 2, 2)
|
||||||
|
b.askOf(t, 3, 3)
|
||||||
|
key, value, ask := holdAsk(t)
|
||||||
|
assigner[key] = value
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
b.refuse(t, ask.ID, roomRefusal)
|
||||||
|
b.deliver(t, warrantFor(first.Ask, "approve"))
|
||||||
|
got := b.askedIDs()
|
||||||
|
if len(got) != 5 || got[4] != ask.ID {
|
||||||
|
t.Fatalf("asks = %v, want the assigner's %s asked again", got, ask.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
b = newBed(t)
|
||||||
|
assigner, _ = withView(b)
|
||||||
|
assigner[key] = value
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
b.refuse(t, ask.ID, "the ask has no headline")
|
||||||
|
if len(b.conds) != 1 || b.conds[0].Subject.Scope != "delivery" || strings.Contains(b.conds[0].Explanation, "merge") {
|
||||||
|
t.Errorf("conditions = %+v", b.conds)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ackFor is the router's warrant for an option of level acknowledge, as the assigner's asks offer.
|
||||||
|
func ackFor(a asks.Ask, option string) asks.Warrant {
|
||||||
|
w := warrantFor(a, option)
|
||||||
|
w.Level, w.Label = asks.Acknowledge, option
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdAskOf is the assigner's hold ask of module plex at the n-th fingerprint, shaped as holdAsk.
|
||||||
|
func holdAskOf(t *testing.T, n int) (string, []byte, asks.Ask) {
|
||||||
|
t.Helper()
|
||||||
|
key, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat(string(rune('0'+n)), 64))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_, _, ask := holdAsk(t)
|
||||||
|
ask.ID = delivery.AskID(key)
|
||||||
|
value, err := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return key, value, ask
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/H5: given seven holds at once (as on 2026-10-25), when they are raised, then at most two are open at the router, a merge
|
||||||
|
// ask is still asked, and each hold answered lets the next waiting one be asked.
|
||||||
|
func TestABurstOfHoldsLeavesRoomForAMergeAsk(t *testing.T) {
|
||||||
|
b := newBed(t)
|
||||||
|
assigner, _ := withView(b)
|
||||||
|
var held []asks.Ask
|
||||||
|
for n := 0; n < 7; n++ {
|
||||||
|
key, value, ask := holdAskOf(t, n)
|
||||||
|
assigner[key] = value
|
||||||
|
b.t.OnAssignerAsk(key)
|
||||||
|
held = append(held, ask)
|
||||||
|
}
|
||||||
|
if got := b.askedIDs(); len(got) != mostAssignerOpen || got[0] != held[0].ID || got[1] != held[1].ID {
|
||||||
|
t.Fatalf("asked %v, want the first %d holds only; log:\n%s", got, mostAssignerOpen, b.logs.String())
|
||||||
|
}
|
||||||
|
_, merge := b.askOf(t, 1, 1)
|
||||||
|
if got := b.askedIDs(); len(got) != mostAssignerOpen+1 || got[mostAssignerOpen] != merge.Ask.ID {
|
||||||
|
t.Fatalf("asked %v, want the merge ask asked beside the holds", got)
|
||||||
|
}
|
||||||
|
if len(b.dropped) != 0 || len(b.conds) != 0 {
|
||||||
|
t.Errorf("router dropped %v, conditions %+v", b.dropped, b.conds)
|
||||||
|
}
|
||||||
|
b.deliver(t, ackFor(held[0], "release"))
|
||||||
|
got := b.askedIDs()
|
||||||
|
if len(got) != mostAssignerOpen+2 || got[len(got)-1] != held[2].ID {
|
||||||
|
t.Fatalf("asked %v after a hold was answered, want the third hold asked next; log:\n%s", got, b.logs.String())
|
||||||
|
}
|
||||||
|
// The merge ask answered frees its own place, and no hold takes it.
|
||||||
|
b.deliver(t, warrantFor(merge.Ask, "approve"))
|
||||||
|
if got := b.askedIDs(); len(got) != mostAssignerOpen+2 {
|
||||||
|
t.Errorf("asked %v after the merge ask was answered, want no further hold", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -524,8 +524,9 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
|
|||||||
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
||||||
t.Fatalf("answer %+v err %v", a, err)
|
t.Fatalf("answer %+v err %v", a, err)
|
||||||
}
|
}
|
||||||
// The view's answer written again changes nothing under the ask.
|
// The view's answer written again with the same option changes nothing under the ask (another option, later, is the
|
||||||
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "keep", "")); err != nil {
|
// operator changing their mind: TestTellerTakesAChangedHoldAnswerFromTheView, issue 1906 note 23).
|
||||||
|
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "")); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
time.Sleep(time.Second)
|
time.Sleep(time.Second)
|
||||||
@@ -545,3 +546,148 @@ func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC
|
|||||||
t.Error("the merge's ask was answered from a view key")
|
t.Error("the merge's ask was answered from a view key")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer starts the program against the bus: the assigner's hold ask, put in
|
||||||
|
// its bucket, is on the operator channel's queue as the teller's ask; the router's warrant for "release" is written under the
|
||||||
|
// ask's key, where the assigner reads it; a second ask the assigner takes away (a hold it lifted) is cancelled at the router
|
||||||
|
// (1906/H1, H2 on the real bus).
|
||||||
|
func TestTellerRaisesAnAssignerAskAndWritesTheOperatorsAnswer(t *testing.T) {
|
||||||
|
now = time.Now().UTC()
|
||||||
|
b := startFull(t)
|
||||||
|
assigner := b.as(t, "n1.mesh-delivery-assigner")
|
||||||
|
broker := filepath.Join(t.TempDir(), "broker")
|
||||||
|
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
||||||
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||||
|
logs := &syncBuffer{}
|
||||||
|
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
||||||
|
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
||||||
|
|
||||||
|
key, value, ask := holdAsk(t)
|
||||||
|
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(500 * time.Millisecond) // the program hears warrants and watches
|
||||||
|
if _, err := askKV.Put(b.ctx, key, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
seat, err := b.js.Stream(b.ctx, "SEAT_OPERATOR_CHANNEL")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var raw *jetstream.RawStreamMsg
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if raw == nil {
|
||||||
|
t.Fatalf("the assigner's ask is not on the seat's queue: %v\n%s", err, logs.String())
|
||||||
|
}
|
||||||
|
var asked asks.Ask
|
||||||
|
if err := json.Unmarshal(raw.Data, &asked); err != nil || asked.ID != ask.ID || asked.Digest() != ask.Digest() {
|
||||||
|
t.Fatalf("the ask on the queue is %+v (%v), want the assigner's", asked, err)
|
||||||
|
}
|
||||||
|
w, _ := json.Marshal(ackFor(ask, "release"))
|
||||||
|
if err := b.ctl.Publish(seatDecided, w); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var e jetstream.KeyValueEntry
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
if e, err = answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if e == nil {
|
||||||
|
t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String())
|
||||||
|
}
|
||||||
|
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID {
|
||||||
|
t.Fatalf("answer %+v err %v", a, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
key2, err := delivery.HoldAskKey("plex", "src2:"+strings.Repeat("3", 64))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ask2 := ask
|
||||||
|
ask2.ID = delivery.AskID(key2)
|
||||||
|
value2, err := delivery.EncodeAssignerAsk(key2, delivery.AssignerAsk{Ask: ask2})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := askKV.Put(b.ctx, key2, value2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
if strings.Contains(logs.String(), ask2.ID+" is asked") {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := askKV.Delete(b.ctx, key2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatCancel); err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err != nil || string(raw.Data) != `{"id":"`+ask2.ID+`"}` {
|
||||||
|
t.Errorf("the ask taken away was not cancelled: %v %v\n%s", raw, err, logs.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTellerTakesAChangedHoldAnswerFromTheView starts the program against the bus: a hold ask answered "keep" earlier, and the
|
||||||
|
// view's answer "release" written now, replace the answer of that asking with "release" (1906/N23 on the real bus).
|
||||||
|
func TestTellerTakesAChangedHoldAnswerFromTheView(t *testing.T) {
|
||||||
|
now = time.Now().UTC()
|
||||||
|
b := startFull(t)
|
||||||
|
assigner, view := b.as(t, "n1.mesh-delivery-assigner"), b.as(t, "n1.mesh-delivery-view")
|
||||||
|
broker := filepath.Join(t.TempDir(), "broker")
|
||||||
|
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
||||||
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
key, value, ask := holdAsk(t)
|
||||||
|
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kept, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: ask.ID + "@" + time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Words: "the operator chose keep"})
|
||||||
|
if _, err := answers.Put(b.ctx, delivery.AnswerKey(key), kept); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
askKV, err := assigner.KeyValue(b.ctx, assignerAsks)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := askKV.Put(b.ctx, key, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||||
|
logs := &syncBuffer{}
|
||||||
|
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
||||||
|
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
||||||
|
viewKV, err := view.KeyValue(b.ctx, viewAnswers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
time.Sleep(500 * time.Millisecond) // the program is watching
|
||||||
|
if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "it was a flake")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
if e, err := answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
|
||||||
|
if a, err := delivery.DecodeAnswer(e.Value()); err == nil && a.Option == "release" && answeredAsking(e.Value()) == ask.ID {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("the kept hold's answer was not changed to release\n%s", logs.String())
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
@@ -107,3 +108,15 @@ func (g kvGate) Get(key string) ([]byte, bool, error) {
|
|||||||
}
|
}
|
||||||
return e.Value(), true, nil
|
return e.Value(), true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetAt is Get with the time the value was written.
|
||||||
|
func (g kvGate) GetAt(key string) ([]byte, time.Time, bool, error) {
|
||||||
|
e, err := g.kv.Get(g.ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return nil, time.Time{}, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, time.Time{}, false, err
|
||||||
|
}
|
||||||
|
return e.Value(), e.Created(), true, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -141,14 +141,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
|
// there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer).
|
||||||
failed := make(chan error, 2)
|
failed := make(chan error, 2)
|
||||||
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
|
assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger)
|
||||||
|
// The assigner's asks are raised on the operator channel whether or not the view's answers can be read (issue 1906).
|
||||||
|
if viewKV != nil {
|
||||||
|
t.View = kvGate{ctx: ctx, kv: viewKV}
|
||||||
|
}
|
||||||
|
if assignerKV != nil {
|
||||||
|
t.Assigner = kvGate{ctx: ctx, kv: assignerKV}
|
||||||
|
go func() {
|
||||||
|
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, t.OnAssignerAskGone, nil)
|
||||||
|
}()
|
||||||
|
}
|
||||||
if assignerKV != nil && viewKV != nil {
|
if assignerKV != nil && viewKV != nil {
|
||||||
t.Assigner, t.View = kvGate{ctx: ctx, kv: assignerKV}, kvGate{ctx: ctx, kv: viewKV}
|
|
||||||
go func() {
|
go func() {
|
||||||
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
|
failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil)
|
||||||
}()
|
}()
|
||||||
go func() {
|
|
||||||
failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, func(string) {}, nil)
|
|
||||||
}()
|
|
||||||
}
|
}
|
||||||
go func() {
|
go func() {
|
||||||
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil {
|
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil {
|
||||||
@@ -170,7 +176,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
|
func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue {
|
||||||
kv, err := js.KeyValue(ctx, name)
|
kv, err := js.KeyValue(ctx, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Printf("the bucket %s cannot be read (%v): the view's answers fill no ask until the teller is started again with it", name, err)
|
logger.Printf("the bucket %s cannot be read (%v): the asks and answers it holds are not carried until the teller is started again with it", name, err)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
return kv
|
return kv
|
||||||
|
|||||||
@@ -340,6 +340,9 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
|
|||||||
if srec == nil {
|
if srec == nil {
|
||||||
return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask)
|
return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask)
|
||||||
}
|
}
|
||||||
|
if isAssignerKey(s.Key) {
|
||||||
|
return t.answerAssigner(s.Key, o, w)
|
||||||
|
}
|
||||||
raw, found, err := t.Gate.Get(s.Key)
|
raw, found, err := t.Gate.Get(s.Key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err)
|
return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err)
|
||||||
@@ -367,26 +370,31 @@ func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Compare-and-set: created when the key has no answer; updated at the revision read when it holds an earlier asking's
|
return t.writeAnswer(s.Key, w.Ask, body, w.Says())
|
||||||
// answer (the gate ignores that one); left when it holds this asking's.
|
}
|
||||||
akey := delivery.AnswerKey(s.Key)
|
|
||||||
|
// writeAnswer writes the answer to the asking askID under the ask's key by compare-and-set: created when the key has no answer;
|
||||||
|
// updated at the revision read when it holds an earlier asking's answer (the reader ignores that one); left when it holds this
|
||||||
|
// asking's.
|
||||||
|
func (t *Teller) writeAnswer(key, askID string, body []byte, words string) error {
|
||||||
|
akey := delivery.AnswerKey(key)
|
||||||
cur, crev, found, err := t.Answers.Get(akey)
|
cur, crev, found, err := t.Answers.Get(akey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the answer under %s could not be read: %w", s.Key, err)
|
return fmt.Errorf("the answer under %s could not be read: %w", key, err)
|
||||||
}
|
}
|
||||||
if found && answeredAsking(cur) == w.Ask {
|
if found && answeredAsking(cur) == askID {
|
||||||
t.logf("%s: an answer is there already, and is not written again", s.Key)
|
t.logf("%s: an answer is there already, and is not written again", key)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
written, err := t.Answers.Write(akey, body, crev, found)
|
written, err := t.Answers.Write(akey, body, crev, found)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("the answer under %s could not be written: %w", s.Key, err)
|
return fmt.Errorf("the answer under %s could not be written: %w", key, err)
|
||||||
}
|
}
|
||||||
if !written {
|
if !written {
|
||||||
t.logf("%s: another writer answered first, and the answer is not written again", s.Key)
|
t.logf("%s: another writer answered first, and the answer is not written again", key)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
t.logf("%s: %s", s.Key, w.Says())
|
t.logf("%s: %s", key, words)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -423,10 +431,10 @@ func (t *Teller) saidOnce(h asker.Handled, w asks.Warrant, err error) {
|
|||||||
t.clear(w.Ask)
|
t.clear(w.Ask)
|
||||||
case h == asker.Refused || (h == asker.Acted && err != nil):
|
case h == asker.Refused || (h == asker.Acted && err != nil):
|
||||||
t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err)
|
t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err)
|
||||||
t.raise(w.Ask, "A merge Approve did not hold", "its answer did not hold here: "+oneLine(err))
|
t.raise(w.Ask, t.kindOf(w.Ask).answerHeadline, "its answer did not hold here: "+oneLine(err))
|
||||||
case h == asker.Recorded && w.Outcome == asks.OutcomeRefused:
|
case h == asker.Recorded && w.Outcome == asks.OutcomeRefused:
|
||||||
t.logf("the router refused the ask %s: %s", w.Ask, w.Words)
|
t.logf("the router refused the ask %s: %s", w.Ask, w.Words)
|
||||||
t.raise(w.Ask, "A merge ask was refused", "the router refused it: "+w.Words)
|
t.raise(w.Ask, t.kindOf(w.Ask).refusedHeadline, "the router refused it: "+w.Words)
|
||||||
case err != nil:
|
case err != nil:
|
||||||
t.logf("the warrant of %s: %v", w.Ask, err)
|
t.logf("the warrant of %s: %v", w.Ask, err)
|
||||||
}
|
}
|
||||||
@@ -446,6 +454,30 @@ func oneLine(err error) string {
|
|||||||
// conditionKey is the key of the condition about an ask.
|
// conditionKey is the key of the condition about an ask.
|
||||||
func conditionKey(id string) string { return delivery.TellerModule + ".merge-ask." + id }
|
func conditionKey(id string) string { return delivery.TellerModule + ".merge-ask." + id }
|
||||||
|
|
||||||
|
// askWords are the words the teller's conditions use for one kind of ask: the gate's merge ask, or one of the assigner's.
|
||||||
|
type askWords struct {
|
||||||
|
scope, noun, needs, nothing, resolved string
|
||||||
|
answerHeadline, refusedHeadline, unaskedHeadline string
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
mergeWords = askWords{scope: "merge", noun: "The merge ask ", needs: "to ask for the Approve again", nothing: "Nothing was merged.",
|
||||||
|
resolved: "A good answer to the merge ask was written.", answerHeadline: "A merge Approve did not hold",
|
||||||
|
refusedHeadline: "A merge ask was refused", unaskedHeadline: "A merge ask could not be asked"}
|
||||||
|
assignerWords = askWords{scope: "delivery", noun: "The delivery ask ", needs: "to answer it in the delivery view",
|
||||||
|
nothing: "Nothing was changed on a machine.", resolved: "A good answer to the delivery ask was written.",
|
||||||
|
answerHeadline: "An answer to a delivery ask did not hold", refusedHeadline: "A delivery ask was refused",
|
||||||
|
unaskedHeadline: "A delivery ask could not be asked"}
|
||||||
|
)
|
||||||
|
|
||||||
|
// kindOf is the words for the ask with this router id, by the key it was asked for.
|
||||||
|
func (t *Teller) kindOf(askID string) askWords {
|
||||||
|
if isAssignerKey(t.condID(askID)) {
|
||||||
|
return assignerWords
|
||||||
|
}
|
||||||
|
return mergeWords
|
||||||
|
}
|
||||||
|
|
||||||
func (t *Teller) raise(askID, headline, why string) {
|
func (t *Teller) raise(askID, headline, why string) {
|
||||||
if t.Say == nil {
|
if t.Say == nil {
|
||||||
return
|
return
|
||||||
@@ -457,11 +489,12 @@ func (t *Teller) raise(askID, headline, why string) {
|
|||||||
}
|
}
|
||||||
t.logf("the condition of %s could not be kept: %v", id, err)
|
t.logf("the condition of %s could not be kept: %v", id, err)
|
||||||
}
|
}
|
||||||
needs := "to ask for the Approve again"
|
k := t.kindOf(askID)
|
||||||
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateRaised,
|
needs := k.needs
|
||||||
Summary: "The merge ask " + id + ": " + why, Headline: headline, Needs: needs,
|
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: k.scope, ID: id}, State: asks.StateRaised,
|
||||||
Explanation: asks.NeedsYou + needs + ". The merge ask " + id + ", " + why + ". Nothing was merged.",
|
Summary: k.noun + id + ": " + why, Headline: headline, Needs: needs,
|
||||||
Resolved: "A good answer to the merge ask was written."}
|
Explanation: asks.NeedsYou + needs + ". " + k.noun + id + ", " + why + ". " + k.nothing,
|
||||||
|
Resolved: k.resolved}
|
||||||
if err := t.Say(c); err != nil {
|
if err := t.Say(c); err != nil {
|
||||||
t.logf("the condition of %s could not be said: %v", id, err)
|
t.logf("the condition of %s could not be said: %v", id, err)
|
||||||
_ = t.Store.Delete(condKey(id))
|
_ = t.Store.Delete(condKey(id))
|
||||||
@@ -477,7 +510,7 @@ func (t *Teller) clear(askID string) {
|
|||||||
if err != nil || e == nil {
|
if err != nil || e == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateCleared}
|
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: t.kindOf(askID).scope, ID: id}, State: asks.StateCleared}
|
||||||
if err := t.Say(c); err != nil {
|
if err := t.Say(c); err != nil {
|
||||||
t.logf("the condition of %s could not be cleared: %v", id, err)
|
t.logf("the condition of %s could not be cleared: %v", id, err)
|
||||||
return
|
return
|
||||||
@@ -578,7 +611,7 @@ func (t *Teller) roomRefused(w asks.Warrant) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
t.logf("the router refused the ask %s for want of room %d times (%s): it is asked no more, and said", w.Ask, n, w.Words)
|
t.logf("the router refused the ask %s for want of room %d times (%s): it is asked no more, and said", w.Ask, n, w.Words)
|
||||||
t.raise(w.Ask, "A merge ask could not be asked", fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
|
t.raise(w.Ask, t.kindOf(w.Ask).unaskedHeadline, fmt.Sprintf("the router refused it %d times for want of room: %s", n, w.Words))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if t.keepWaiting(ws, rec) != nil {
|
if t.keepWaiting(ws, rec) != nil {
|
||||||
@@ -675,6 +708,11 @@ func (t *Teller) askWaiting(n int) {
|
|||||||
i++
|
i++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
assigner := isAssignerKey(x.Key)
|
||||||
|
if assigner && !t.assignerRoom() {
|
||||||
|
i++ // the assigner's asks hold their most open: it waits, and a merge ask after it may be asked
|
||||||
|
continue
|
||||||
|
}
|
||||||
a, why := t.stillAsked(x)
|
a, why := t.stillAsked(x)
|
||||||
if why != "" {
|
if why != "" {
|
||||||
t.logf("%s: the ask %s, waiting for room, is not asked again: %s", x.Key, x.ID, why)
|
t.logf("%s: the ask %s, waiting for room, is not asked again: %s", x.Key, x.ID, why)
|
||||||
@@ -687,6 +725,9 @@ func (t *Teller) askWaiting(n int) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
ws[i].Asked = true
|
ws[i].Asked = true
|
||||||
|
if assigner {
|
||||||
|
t.assignerAsked(x.ID)
|
||||||
|
}
|
||||||
t.logf("%s: the ask %s is asked of the operator again (refused %d times for want of room)", x.Key, x.ID, x.Refusals)
|
t.logf("%s: the ask %s is asked of the operator again (refused %d times for want of room)", x.Key, x.ID, x.Refusals)
|
||||||
if n--; n <= 0 {
|
if n--; n <= 0 {
|
||||||
return
|
return
|
||||||
@@ -701,6 +742,9 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
|
|||||||
if x.Key == "" {
|
if x.Key == "" {
|
||||||
return asks.Ask{}, "the teller holds no gate key for it"
|
return asks.Ask{}, "the teller holds no gate key for it"
|
||||||
}
|
}
|
||||||
|
if isAssignerKey(x.Key) {
|
||||||
|
return t.stillAskedByAssigner(x)
|
||||||
|
}
|
||||||
raw, found, err := t.Gate.Get(x.Key)
|
raw, found, err := t.Gate.Get(x.Key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return asks.Ask{}, "the gate's ask could not be read: " + err.Error()
|
return asks.Ask{}, "the gate's ask could not be read: " + err.Error()
|
||||||
@@ -724,6 +768,26 @@ func (t *Teller) stillAsked(x waitingAsk) (asks.Ask, string) {
|
|||||||
return ga.Ask, ""
|
return ga.Ask, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stillAskedByAssigner is stillAsked for one of the assigner's asks.
|
||||||
|
func (t *Teller) stillAskedByAssigner(x waitingAsk) (asks.Ask, string) {
|
||||||
|
a, found, err := t.assignerAsk(x.Key)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
return asks.Ask{}, err.Error()
|
||||||
|
case !found:
|
||||||
|
return asks.Ask{}, "the assigner has taken it away"
|
||||||
|
case a.Ask.ID != x.ID:
|
||||||
|
return asks.Ask{}, "the assigner asks again under " + a.Ask.ID
|
||||||
|
}
|
||||||
|
if cur, _, has, err := t.Answers.Get(delivery.AnswerKey(x.Key)); err == nil && has && answeredAsking(cur) == x.ID {
|
||||||
|
return asks.Ask{}, "it is answered"
|
||||||
|
}
|
||||||
|
if err := a.Ask.Check(t.now()); err != nil {
|
||||||
|
return asks.Ask{}, "it no longer holds: " + err.Error()
|
||||||
|
}
|
||||||
|
return a.Ask, ""
|
||||||
|
}
|
||||||
|
|
||||||
// askAgain keeps the ask afresh as the asker would (its record of the refusal is replaced, so its warrant is taken) and
|
// askAgain keeps the ask afresh as the asker would (its record of the refusal is replaced, so its warrant is taken) and
|
||||||
// publishes it under the bus message id given.
|
// publishes it under the bus message id given.
|
||||||
func (t *Teller) askAgain(a asks.Ask, msgID string) error {
|
func (t *Teller) askAgain(a asks.Ask, msgID string) error {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
@@ -25,7 +26,8 @@ func (t *Teller) OnViewAnswer(key string) {
|
|||||||
t.fillFromView(askKey)
|
t.fillFromView(askKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now.
|
// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now, and an
|
||||||
|
// ask not answered yet is raised on the operator channel (issue 1906; assignerasks.go).
|
||||||
func (t *Teller) OnAssignerAsk(key string) {
|
func (t *Teller) OnAssignerAsk(key string) {
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
defer t.mu.Unlock()
|
defer t.mu.Unlock()
|
||||||
@@ -34,10 +36,14 @@ func (t *Teller) OnAssignerAsk(key string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
t.fillFromView(key)
|
t.fillFromView(key)
|
||||||
|
t.raiseAssignerAsk(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// fillFromView writes the answer to the assigner's ask at askKey from the view's answer, when both are there and readable, the
|
// fillFromView writes the answer to the assigner's ask at askKey from the view's answer, when both are there and readable, the
|
||||||
// ask is of a kind that needs no proof, the option is one the ask offers and the asking is not answered yet.
|
// ask is of a kind that needs no proof, the option is one the ask offers and the asking is not answered yet. A hold ask answered
|
||||||
|
// already takes a later answer of the view that chose another option: the operator changed their mind (issue 1906 note 23: a
|
||||||
|
// "keep" given on the phone could not be undone). Later means the view's answer was written after the answer that stands, so a
|
||||||
|
// view answer read again at a start never replaces a newer answer from the phone.
|
||||||
func (t *Teller) fillFromView(askKey string) {
|
func (t *Teller) fillFromView(askKey string) {
|
||||||
if t.Assigner == nil || t.View == nil {
|
if t.Assigner == nil || t.View == nil {
|
||||||
return
|
return
|
||||||
@@ -47,7 +53,7 @@ func (t *Teller) fillFromView(askKey string) {
|
|||||||
t.logf("%s is not answered from the view: %v", askKey, err)
|
t.logf("%s is not answered from the view: %v", askKey, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
rawAnswer, found, err := t.View.Get(viewKey)
|
rawAnswer, at, found, err := t.viewAnswer(viewKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.logf("%s: the view's answer could not be read, so the ask is left: %v", askKey, err)
|
t.logf("%s: the view's answer could not be read, so the ask is left: %v", askKey, err)
|
||||||
return
|
return
|
||||||
@@ -83,14 +89,14 @@ func (t *Teller) fillFromView(askKey string) {
|
|||||||
t.logf("%s: the answer could not be read: %v", askKey, err)
|
t.logf("%s: the answer could not be read: %v", askKey, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if has && answeredAsking(cur) == ask.Ask.ID {
|
if has && answeredAsking(cur) == ask.Ask.ID && !t.changesMind(askKey, cur, va, at) {
|
||||||
return // this asking is answered, from the view or elsewhere
|
return // this asking is answered, from the view or elsewhere
|
||||||
}
|
}
|
||||||
words := "answered in the view (mesh-delivery-view): " + va.Option
|
words := "answered in the view (mesh-delivery-view): " + va.Option
|
||||||
if w := strings.TrimSpace(va.Words); w != "" {
|
if w := strings.TrimSpace(va.Words); w != "" {
|
||||||
words += ": " + w
|
words += ": " + w
|
||||||
}
|
}
|
||||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: va.Option, Warrant: ask.Ask.ID + "@" + t.now().UTC().Format("2006-01-02T15:04:05.999999999Z07:00"), Words: words})
|
body, err := delivery.EncodeAnswer(delivery.Answer{Option: va.Option, Warrant: ask.Ask.ID + "@" + at.UTC().Format(time.RFC3339Nano), Words: words})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.logf("%s: the answer could not be made: %v", askKey, err)
|
t.logf("%s: the answer could not be made: %v", askKey, err)
|
||||||
return
|
return
|
||||||
@@ -103,5 +109,48 @@ func (t *Teller) fillFromView(askKey string) {
|
|||||||
t.logf("%s: another writer answered first, and the view's answer is not written", askKey)
|
t.logf("%s: another writer answered first, and the view's answer is not written", askKey)
|
||||||
default:
|
default:
|
||||||
t.logf("%s: %s", askKey, words)
|
t.logf("%s: %s", askKey, words)
|
||||||
|
t.cancelOpen(askKey, ask.Ask.ID) // answered in the view: the same ask on the phone is taken back
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// timedAnswers is the view's bucket as the program reads it: each answer with the time it was written.
|
||||||
|
type timedAnswers interface {
|
||||||
|
GetAt(key string) (value []byte, at time.Time, found bool, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// viewAnswer is the view's answer at key and when it was written; a bucket that keeps no time gives the teller's now.
|
||||||
|
func (t *Teller) viewAnswer(key string) ([]byte, time.Time, bool, error) {
|
||||||
|
if tv, ok := t.View.(timedAnswers); ok {
|
||||||
|
v, at, found, err := tv.GetAt(key)
|
||||||
|
if err == nil && !at.IsZero() {
|
||||||
|
return v, at, found, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, time.Time{}, false, err
|
||||||
|
}
|
||||||
|
return v, t.now(), found, nil
|
||||||
|
}
|
||||||
|
v, found, err := t.View.Get(key)
|
||||||
|
return v, t.now(), found, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// changesMind says whether the view's answer va, written at at, replaces the answer cur of the same asking of the hold ask at
|
||||||
|
// askKey: it chose another option and was written after cur's warrant.
|
||||||
|
func (t *Teller) changesMind(askKey string, cur []byte, va delivery.ViewAnswer, at time.Time) bool {
|
||||||
|
k, err := delivery.ParseAnyAskKey(askKey)
|
||||||
|
if err != nil || k.Kind != delivery.AskKindHold {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
a, err := delivery.DecodeAnswer(cur)
|
||||||
|
if err != nil || a.Option == va.Option {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, when, _ := strings.Cut(a.Warrant, "@")
|
||||||
|
before, err := time.Parse(time.RFC3339Nano, when)
|
||||||
|
if err != nil || !at.After(before) {
|
||||||
|
t.logf("%s: the view's answer %s is not newer than the answer %s that stands (%s), and is not taken", askKey, va.Option, a.Option, when)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
t.logf("%s: the operator changed the answer from %s to %s in the view", askKey, a.Option, va.Option)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package main
|
|||||||
import (
|
import (
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -10,8 +11,9 @@ import (
|
|||||||
|
|
||||||
const fpHeld = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
const fpHeld = "src2:2222222222222222222222222222222222222222222222222222222222222222"
|
||||||
|
|
||||||
// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, as a fixture (in release 5 the assigner writes
|
// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, shaped as the assigner composes it
|
||||||
// none yet; the value is `{"v":1,"ask":<the router's ask>}`).
|
// (mesh-delivery-assigner internal/assign asks.go HoldAsk: two options of level acknowledge that bind no act, so the router
|
||||||
|
// takes it; the value is `{"v":1,"ask":<the router's ask>}`).
|
||||||
func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
key, err := delivery.HoldAskKey("plex", fpHeld)
|
key, err := delivery.HoldAskKey("plex", fpHeld)
|
||||||
@@ -20,7 +22,7 @@ func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) {
|
|||||||
}
|
}
|
||||||
ask = asks.Ask{ID: delivery.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: asks.Operator,
|
ask = asks.Ask{ID: delivery.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: asks.Operator,
|
||||||
Expires: now.Add(asks.ApproveLasts), OnExpiry: "the fingerprint stays held",
|
Expires: now.Add(asks.ApproveLasts), OnExpiry: "the fingerprint stays held",
|
||||||
Options: []asks.Option{{ID: "release", Label: "Release", Does: "lets the move go on", Level: asks.Approve}, {ID: "keep", Label: "Keep held", Does: "keeps it held", Level: asks.Approve}}}
|
Options: []asks.Option{{ID: "keep", Label: "Keep it held", Does: "keeps it held", Level: asks.Acknowledge}, {ID: "release", Label: "Release it", Does: "lets the move go on", Level: asks.Acknowledge}}}
|
||||||
value, err = delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
value, err = delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -117,8 +119,11 @@ func TestAViewAnswerOfAnOptionTheAskDoesNotOfferFillsNothing(t *testing.T) { //
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A view answer written before the answer that stands never replaces it (1906/N23: only a later one may, for a hold). The
|
||||||
|
// view's time is fixed here, not the package's clock, which the integration tests set to the wall clock.
|
||||||
func TestAViewAnswerNeverReplacesTheAnswerOfTheSameAsking(t *testing.T) { // 1297/B10
|
func TestAViewAnswerNeverReplacesTheAnswerOfTheSameAsking(t *testing.T) { // 1297/B10
|
||||||
b := newBed(t)
|
b := newBed(t)
|
||||||
|
b.t.Now = func() time.Time { return time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }
|
||||||
assigner, view := withView(b)
|
assigner, view := withView(b)
|
||||||
key, value, ask := holdAsk(t)
|
key, value, ask := holdAsk(t)
|
||||||
assigner[key] = value
|
assigner[key] = value
|
||||||
@@ -174,3 +179,53 @@ func TestAStaleViewAnswerOfAnotherAskingFillsNothing(t *testing.T) { // 1297/B10
|
|||||||
t.Errorf("writes %d; log:\n%s", b.answers.writes, b.logs.String())
|
t.Errorf("writes %d; log:\n%s", b.answers.writes, b.logs.String())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// timedView is the view's bucket with the time each answer was written.
|
||||||
|
type timedView struct {
|
||||||
|
memGate
|
||||||
|
at map[string]time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v timedView) GetAt(key string) ([]byte, time.Time, bool, error) {
|
||||||
|
b, ok := v.memGate[key]
|
||||||
|
return b, v.at[key], ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/N23-AC4: given a hold ask answered "keep" on the phone, when the operator answers "release" in the view later, then the
|
||||||
|
// teller replaces the answer of that asking with "release", whose warrant names the asking and the view's time; a view answer
|
||||||
|
// written before the phone's answer, the same option, or an ask of another kind replaces nothing.
|
||||||
|
func TestALaterViewAnswerChangesAHoldsAnswer(t *testing.T) {
|
||||||
|
setup := func(viewAt time.Time, option string) (*bed, string, asks.Ask) {
|
||||||
|
b := newBed(t)
|
||||||
|
assigner := memGate{}
|
||||||
|
view := timedView{memGate: memGate{}, at: map[string]time.Time{}}
|
||||||
|
b.t.Assigner, b.t.View = assigner, view
|
||||||
|
key, value, ask := holdAsk(t)
|
||||||
|
assigner[key] = value
|
||||||
|
kept, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: ask.ID + "@" + now.Add(time.Hour).UTC().Format(time.RFC3339Nano), Words: "the operator, via telegram, chose keep"})
|
||||||
|
b.answers.written[key], b.answers.revs[key] = kept, 3
|
||||||
|
view.memGate[key+".view"] = viewAnswer(t, option, "it was a flake")
|
||||||
|
view.at[key+".view"] = viewAt
|
||||||
|
b.t.OnViewAnswer(key + ".view")
|
||||||
|
return b, key, ask
|
||||||
|
}
|
||||||
|
later := now.Add(2 * time.Hour)
|
||||||
|
b, key, ask := setup(later, "release")
|
||||||
|
a, err := delivery.DecodeAnswer(b.answers.written[key])
|
||||||
|
if err != nil || b.answers.writes != 1 || a.Option != "release" || a.Warrant != ask.ID+"@"+later.UTC().Format(time.RFC3339Nano) {
|
||||||
|
t.Fatalf("writes %d, answer %+v, %v; log:\n%s", b.answers.writes, a, err, b.logs.String())
|
||||||
|
}
|
||||||
|
for name, b := range map[string]*bed{
|
||||||
|
"older": func() *bed { b, _, _ := setup(now, "release"); return b }(),
|
||||||
|
"same": func() *bed { b, _, _ := setup(later, "keep"); return b }(),
|
||||||
|
} {
|
||||||
|
if b.answers.writes != 0 {
|
||||||
|
t.Errorf("%s: the answer was replaced", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A restore ask answered stays answered: only a hold's answer may be changed.
|
||||||
|
bb := newBed(t)
|
||||||
|
if bb.t.changesMind("restore.ace.plex.2222222222222222222222222222222222222222222222222222222222222222", b.answers.written[key], delivery.ViewAnswer{Option: "keep"}, later.Add(time.Hour)) {
|
||||||
|
t.Error("a restore ask's answer was changed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module mesh-delivery-teller
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
// Command mesh-delivery-cli is the operator's command for the delivery view (novox/hq ADR 0429 §3). It asks the
|
// Command mesh-delivery-cli is the operator's command for the delivery view (novox/hq ADR 0429 §3). It asks the
|
||||||
// view's local socket for the rows and prints them; it holds no credential. `answer <ask key> <option> [words]` asks the view
|
// view's local socket for the rows and prints them; it holds no credential. `answer <ask key> <option> [words]` asks the view
|
||||||
// to write the operator's answer to an ask that needs no proof of who answered (the view refuses the merge's Approve).
|
// to write the operator's answer to an ask that needs no proof of who answered (the view refuses the merge's Approve). `holds`
|
||||||
|
// lists every fingerprint the assigner holds with its node, cause and answer; `hold-release <module> <fingerprint> [words]` and
|
||||||
|
// `hold-keep <module> <fingerprint> [words]` answer its hold ask, again when the operator changes their mind (issue 1906).
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -10,6 +12,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
"text/tabwriter"
|
||||||
|
|
||||||
"mesh-delivery-view/internal/view"
|
"mesh-delivery-view/internal/view"
|
||||||
)
|
)
|
||||||
@@ -31,9 +34,26 @@ func run(args []string, socket string, out, errOut io.Writer) int {
|
|||||||
if err := fs.Parse(args); err != nil {
|
if err := fs.Parse(args); err != nil {
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
|
if fs.NArg() > 0 && fs.Arg(0) == "holds" {
|
||||||
|
return holds(socket, *asJSON, out, errOut)
|
||||||
|
}
|
||||||
|
if fs.NArg() > 0 && (fs.Arg(0) == "hold-release" || fs.Arg(0) == "hold-keep") {
|
||||||
|
if fs.NArg() < 3 {
|
||||||
|
fmt.Fprintf(errOut, "mesh-delivery-cli %s <module> <fingerprint> [words]\n", fs.Arg(0))
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
option := strings.TrimPrefix(fs.Arg(0), "hold-")
|
||||||
|
r, err := view.AnswerHoldOn(socket, fs.Arg(1), fs.Arg(2), option, strings.Join(fs.Args()[3:], " "))
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
fmt.Fprintf(out, "answered the hold of %s at %s with %s; the teller fills the ask from it and the assigner reads it\n", r.Module, r.FP, option)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
if fs.NArg() > 0 {
|
if fs.NArg() > 0 {
|
||||||
if fs.Arg(0) != "answer" || fs.NArg() < 3 {
|
if fs.Arg(0) != "answer" || fs.NArg() < 3 {
|
||||||
fmt.Fprintln(errOut, "mesh-delivery-cli shows the rows, or answers: mesh-delivery-cli answer <ask key> <option> [words]")
|
fmt.Fprintln(errOut, "mesh-delivery-cli shows the rows, or answers: mesh-delivery-cli answer <ask key> <option> [words]; holds; hold-release|hold-keep <module> <fingerprint> [words]")
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
key, option := fs.Arg(1), fs.Arg(2)
|
key, option := fs.Arg(1), fs.Arg(2)
|
||||||
@@ -61,3 +81,53 @@ func run(args []string, socket string, out, errOut io.Writer) int {
|
|||||||
view.Table(out, s)
|
view.Table(out, s)
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// holds prints every hold: module, fingerprint, node, since, answer, words.
|
||||||
|
func holds(socket string, asJSON bool, out, errOut io.Writer) int {
|
||||||
|
rows, err := view.AskHolds(socket)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
if asJSON {
|
||||||
|
enc := json.NewEncoder(out)
|
||||||
|
enc.SetIndent("", " ")
|
||||||
|
if err := enc.Encode(rows); err != nil {
|
||||||
|
fmt.Fprintln(errOut, "mesh-delivery-cli:", err)
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if len(rows) == 0 {
|
||||||
|
fmt.Fprintln(out, "the assigner holds nothing")
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
tw := tabwriter.NewWriter(out, 0, 2, 2, ' ', 0)
|
||||||
|
fmt.Fprintln(tw, "MODULE\tFINGERPRINT\tNODE\tSINCE\tANSWER\tWORDS")
|
||||||
|
for _, r := range rows {
|
||||||
|
answer := r.Answer
|
||||||
|
switch {
|
||||||
|
case r.AskKey == "":
|
||||||
|
answer = "no ask"
|
||||||
|
case r.Pending != "":
|
||||||
|
answer = r.Pending + " (from the view, the teller fills it)"
|
||||||
|
case answer == "":
|
||||||
|
answer = "not answered"
|
||||||
|
}
|
||||||
|
fp := strings.TrimPrefix(r.FP, "src2:")
|
||||||
|
if len(fp) > 12 {
|
||||||
|
fp = fp[:12]
|
||||||
|
}
|
||||||
|
since := ""
|
||||||
|
if !r.Since.IsZero() {
|
||||||
|
since = r.Since.Local().Format("2006-01-02 15:04")
|
||||||
|
}
|
||||||
|
words := r.Words
|
||||||
|
if r.Error != "" {
|
||||||
|
words = "unreadable: " + r.Error
|
||||||
|
}
|
||||||
|
fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", r.Module, fp, r.Node, since, answer, words)
|
||||||
|
}
|
||||||
|
tw.Flush()
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|||||||
@@ -240,3 +240,46 @@ func TestTheAnswerCommandAnswersAnAskNeedingNoProof(t *testing.T) { // 1297/B9b
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1906/N23-AC3: `holds` lists the holds with node, cause and answer; `hold-release <module> <fingerprint>` and `hold-keep` answer
|
||||||
|
// the hold ask through the view, also one answered already.
|
||||||
|
func TestTheHoldCommandsListAndAnswerHolds(t *testing.T) {
|
||||||
|
fp := "src2:04efd1a1aa" + strings.Repeat("2", 54)
|
||||||
|
key, err := delivery.HoldAskKey("systemd-resolved", fp)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
hold := asks.Ask{ID: delivery.AskID(key), Headline: "systemd-resolved is held", Explanation: "ace failed the check.", Who: asks.Operator, Expires: time.Now().Add(time.Hour), OnExpiry: "stays held",
|
||||||
|
Options: []asks.Option{{ID: "keep", Label: "Keep", Does: "stays", Level: asks.Acknowledge}, {ID: "release", Label: "Release", Does: "goes on", Level: asks.Acknowledge}}}
|
||||||
|
value, _ := delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: hold})
|
||||||
|
hkey, _ := delivery.HeldKey("systemd-resolved", fp)
|
||||||
|
held, _ := delivery.EncodeHeld(delivery.Held{Node: "ace", Words: "health failed, phase running: dns timeout"})
|
||||||
|
ans, _ := delivery.EncodeAnswer(delivery.Answer{Option: "keep", Warrant: hold.ID + "@2026-10-25T04:00:00Z", Words: "the operator chose keep"})
|
||||||
|
m, w := view.New(), &putter{puts: map[string][]byte{}}
|
||||||
|
m.SetWriter(w)
|
||||||
|
m.Apply(view.Entry{Bucket: delivery.AssignerAsksBucket, Key: key, Value: value})
|
||||||
|
m.Apply(view.Entry{Bucket: delivery.AssignerWantsBucket, Key: hkey, Value: held})
|
||||||
|
m.Apply(view.Entry{Bucket: delivery.AnswersBucket, Key: key, Value: ans})
|
||||||
|
socket := serveWith(t, m)
|
||||||
|
|
||||||
|
var out, errOut bytes.Buffer
|
||||||
|
if code := run([]string{"holds"}, socket, &out, &errOut); code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||||
|
}
|
||||||
|
for _, want := range []string{"systemd-resolved", "04efd1a1aa22", "ace", "keep", "dns timeout"} {
|
||||||
|
if !strings.Contains(out.String(), want) {
|
||||||
|
t.Errorf("holds printed %q, missing %q", out.String(), want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.Reset()
|
||||||
|
if code := run([]string{"hold-release", "systemd-resolved", "04efd1a1", "a", "flake"}, socket, &out, &errOut); code != 0 {
|
||||||
|
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||||
|
}
|
||||||
|
if a, err := delivery.DecodeViewAnswer(w.puts[key+".view"]); err != nil || a.Option != "release" || a.Asking != hold.ID || a.Words != "a flake" {
|
||||||
|
t.Errorf("the view's answer = %+v, %v", a, err)
|
||||||
|
}
|
||||||
|
errOut.Reset()
|
||||||
|
if code := run([]string{"hold-keep", "systemd-resolved"}, socket, &out, &errOut); code != 2 {
|
||||||
|
t.Errorf("hold-keep without a fingerprint: exit %d", code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ func Tools(r view.Reader) []stdio.Tool {
|
|||||||
}},
|
}},
|
||||||
{Name: view.BuilderBuildReadTool,
|
{Name: view.BuilderBuildReadTool,
|
||||||
Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " +
|
Description: "One build of the builder as stored. Give the module and its fingerprint (fp2, src2:...), or the module alone for the build " +
|
||||||
"at the fingerprint the indexer holds for it now, joined through modules.<module>; with commit, only if the indexer holds the module at that commit. Read-only.",
|
"at the fingerprint the indexer holds for it now, joined through modules.<module>; with commit, only if the indexer holds the module at that commit (a module's value names the commit its fingerprint first appeared at, and stays there through merges that leave the module alone). Read-only.",
|
||||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||||
"module": map[string]any{"type": "string", "description": "the module's name"},
|
"module": map[string]any{"type": "string", "description": "the module's name"},
|
||||||
"fp2": map[string]any{"type": "string", "description": "its new-form fingerprint; empty means the indexer's current one"},
|
"fp2": map[string]any{"type": "string", "description": "its new-form fingerprint; empty means the indexer's current one"},
|
||||||
|
|||||||
@@ -0,0 +1,188 @@
|
|||||||
|
package view
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The assigner's holds, listed and answered from the operator's command (issue 1906 note 23). A hold ask answered once (on the
|
||||||
|
// phone or in the view) ended with that answer, so nothing listed it any more and an answer "keep" could not be changed: the
|
||||||
|
// operator kept systemd-resolved held and then could not release it, and a kept hold is never lifted by itself. Now the view
|
||||||
|
// lists every hold with its node, its cause and its answer, and answers the hold ask that stands, again if the operator changes
|
||||||
|
// their mind, through the view's one write (`<ask key>.view`), which the teller takes into the ask's answer. Nothing else writes:
|
||||||
|
// the answer still reaches the assigner only through the teller (mesh-delivery-teller viewanswers.go).
|
||||||
|
|
||||||
|
// The options of the hold ask (mesh-delivery-assigner internal/assign asks.go HoldAsk).
|
||||||
|
const (
|
||||||
|
HoldRelease = "release"
|
||||||
|
HoldKeep = "keep"
|
||||||
|
)
|
||||||
|
|
||||||
|
type heldEntry struct {
|
||||||
|
module, fp string
|
||||||
|
value delivery.Held
|
||||||
|
err string
|
||||||
|
at time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyHeld takes a change of a held key of the assigner's wants, and says whether the key was one.
|
||||||
|
func (m *Model) applyHeld(e Entry) bool {
|
||||||
|
module, fp, err := delivery.ParseHeldKey(e.Key)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if e.Deleted {
|
||||||
|
delete(m.holds, e.Key)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
h, err := delivery.DecodeHeld(e.Value)
|
||||||
|
m.holds[e.Key] = heldEntry{module: module, fp: fp, value: h, err: errText(err), at: e.At}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// HoldRow is one fingerprint the assigner holds: the module, the fingerprint, the node whose health check failed, the hold's
|
||||||
|
// words (its cause) and since when; the ask about it (its key and the asking's id, empty when the assigner asks none), and the
|
||||||
|
// answer that stands for that asking (the option and its words, empty when none), with the view's own answer when it waits for
|
||||||
|
// the teller.
|
||||||
|
type HoldRow struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
FP string `json:"fp"`
|
||||||
|
Node string `json:"node,omitempty"`
|
||||||
|
Words string `json:"words,omitempty"`
|
||||||
|
Since time.Time `json:"since,omitempty"`
|
||||||
|
AskKey string `json:"ask-key,omitempty"`
|
||||||
|
Asking string `json:"asking,omitempty"`
|
||||||
|
Answer string `json:"answer,omitempty"`
|
||||||
|
Answered string `json:"answered,omitempty"`
|
||||||
|
Pending string `json:"pending,omitempty"`
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Holds is every hold, ordered by module and fingerprint.
|
||||||
|
func (m *Model) Holds() []HoldRow {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
out := []HoldRow{}
|
||||||
|
for _, h := range m.holds {
|
||||||
|
row := HoldRow{Module: h.module, FP: h.fp, Node: h.value.Node, Words: h.value.Words, Since: h.at, Error: h.err}
|
||||||
|
if key, err := delivery.HoldAskKey(h.module, h.fp); err == nil {
|
||||||
|
if a, ok := m.aasks[key]; ok && a.err == "" {
|
||||||
|
row.AskKey, row.Asking = key, a.ask.ID
|
||||||
|
if ans, ok := m.answerOf[key]; ok && m.answers[key] == a.ask.ID {
|
||||||
|
row.Answer, row.Answered = ans.Option, ans.Words
|
||||||
|
}
|
||||||
|
if c := m.chosenOf(key, a.ask); c != "" && c != row.Answer {
|
||||||
|
row.Pending = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out = append(out, row)
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if out[i].Module != out[j].Module {
|
||||||
|
return out[i].Module < out[j].Module
|
||||||
|
}
|
||||||
|
return out[i].FP < out[j].FP
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// AnswerHold writes the view's answer to the hold ask of module at the fingerprint fp: release or keep, with the operator's
|
||||||
|
// words. fp is the whole fingerprint, or the start of its hex (as the rows print it), naming one hold of the module. Unlike
|
||||||
|
// Answer it answers an ask answered already, so the operator can change their mind; an answer equal to the one that stands is
|
||||||
|
// refused, as is a hold the assigner asks nothing about.
|
||||||
|
func (m *Model) AnswerHold(module, fp, option, words string) (HoldRow, error) {
|
||||||
|
option = strings.TrimSpace(option)
|
||||||
|
if option != HoldRelease && option != HoldKeep {
|
||||||
|
return HoldRow{}, fmt.Errorf("a hold is answered %s or %s, not %q", HoldRelease, HoldKeep, option)
|
||||||
|
}
|
||||||
|
var match []HoldRow
|
||||||
|
want := strings.TrimPrefix(strings.TrimSpace(fp), delivery.SourcePrefix2)
|
||||||
|
for _, r := range m.Holds() {
|
||||||
|
if r.Module == module && want != "" && strings.HasPrefix(strings.TrimPrefix(r.FP, delivery.SourcePrefix2), want) {
|
||||||
|
match = append(match, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case len(match) == 0:
|
||||||
|
return HoldRow{}, fmt.Errorf("the assigner holds no fingerprint %q of %s (mesh-delivery-cli holds lists them)", fp, module)
|
||||||
|
case len(match) > 1:
|
||||||
|
return HoldRow{}, fmt.Errorf("%q names %d holds of %s: give more of the fingerprint", fp, len(match), module)
|
||||||
|
}
|
||||||
|
r := match[0]
|
||||||
|
if r.AskKey == "" {
|
||||||
|
return r, fmt.Errorf("the assigner asks nothing about the hold of %s at %s, so there is nothing to answer", module, r.FP)
|
||||||
|
}
|
||||||
|
if r.Answer == option && r.Pending == "" {
|
||||||
|
return r, fmt.Errorf("the hold of %s at %s is answered %s already", module, r.FP, option)
|
||||||
|
}
|
||||||
|
m.mu.Lock()
|
||||||
|
a, w := m.aasks[r.AskKey], m.writer
|
||||||
|
m.mu.Unlock()
|
||||||
|
if !slices.ContainsFunc(a.ask.Options, func(o asks.Option) bool { return o.ID == option }) {
|
||||||
|
return r, fmt.Errorf("the hold ask %s offers no option %q", r.AskKey, option)
|
||||||
|
}
|
||||||
|
if w == nil {
|
||||||
|
return r, fmt.Errorf("the view cannot write its answers here")
|
||||||
|
}
|
||||||
|
key, err := delivery.ViewAnswerKey(r.AskKey)
|
||||||
|
if err != nil {
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
body, err := delivery.EncodeViewAnswer(delivery.ViewAnswer{Asking: a.ask.ID, Option: option, Words: strings.TrimSpace(words)})
|
||||||
|
if err != nil {
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
if err := w.Put(key, body); err != nil {
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// askOn sends one ask to the socket at path and decodes the reply into out; the view's refusal is the error.
|
||||||
|
func askOn(path string, req ask, out any) error {
|
||||||
|
c, err := net.DialTimeout("unix", path, 5*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot reach the view at %s: %w", path, err)
|
||||||
|
}
|
||||||
|
defer c.Close()
|
||||||
|
c.SetDeadline(time.Now().Add(10 * time.Second))
|
||||||
|
raw, _ := json.Marshal(req)
|
||||||
|
if _, err := c.Write(append(raw, '\n')); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
var reply json.RawMessage
|
||||||
|
if err := json.NewDecoder(c).Decode(&reply); err != nil {
|
||||||
|
return fmt.Errorf("the view's reply cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
var refusal struct {
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(reply, &refusal) == nil && refusal.Error != "" {
|
||||||
|
return errors.New(refusal.Error)
|
||||||
|
}
|
||||||
|
return json.Unmarshal(reply, out)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AskHolds asks the socket at path for every hold.
|
||||||
|
func AskHolds(path string) ([]HoldRow, error) {
|
||||||
|
var out []HoldRow
|
||||||
|
err := askOn(path, ask{Ask: "holds"}, &out)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// AnswerHoldOn asks the socket at path to answer the hold ask of module at fp with option.
|
||||||
|
func AnswerHoldOn(path, module, fp, option, words string) (HoldRow, error) {
|
||||||
|
var out HoldRow
|
||||||
|
err := askOn(path, ask{Ask: "hold", Module: module, FP: fp, Option: option, Words: words}, &out)
|
||||||
|
return out, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package view
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// heldE is the assigner's hold of module at fp, as it writes it in its wants.
|
||||||
|
func heldE(t *testing.T, module, fp, node, words string) Entry {
|
||||||
|
t.Helper()
|
||||||
|
key, err := delivery.HeldKey(module, fp)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
v, err := delivery.EncodeHeld(delivery.Held{Node: node, Words: words})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return Entry{Bucket: delivery.AssignerWantsBucket, Key: key, Value: v, At: t0}
|
||||||
|
}
|
||||||
|
|
||||||
|
func answered(t *testing.T, key, id, option string) Entry {
|
||||||
|
t.Helper()
|
||||||
|
v, err := delivery.EncodeAnswer(delivery.Answer{Option: option, Warrant: id + "@2026-10-25T04:00:00Z", Words: "the operator, via telegram, chose " + option})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return Entry{Bucket: delivery.AnswersBucket, Key: key, Value: v, At: t0}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/N23-AC1: given holds in the assigner's wants, the view lists each with its node, its cause in words, its hold ask and
|
||||||
|
// the answer that stands for that asking; a hold without an ask says so, and a held key is never taken for a want.
|
||||||
|
func TestTheViewListsEveryHoldWithItsNodeCauseAndAnswer(t *testing.T) {
|
||||||
|
m := New()
|
||||||
|
e, a := holdAsk(t)
|
||||||
|
fill(m, e, heldE(t, "plex", fpNew, "ace", "health failed, phase running: connection refused"), answered(t, e.Key, a.ID, "keep"),
|
||||||
|
heldE(t, "web", fpNew, "g14", "health failed"))
|
||||||
|
got := m.Holds()
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("holds = %+v", got)
|
||||||
|
}
|
||||||
|
p := got[0]
|
||||||
|
if p.Module != "plex" || p.FP != fpNew || p.Node != "ace" || !strings.Contains(p.Words, "connection refused") || p.AskKey != e.Key || p.Asking != a.ID || p.Answer != "keep" {
|
||||||
|
t.Errorf("plex = %+v", p)
|
||||||
|
}
|
||||||
|
if w := got[1]; w.Module != "web" || w.Node != "g14" || w.AskKey != "" || w.Answer != "" {
|
||||||
|
t.Errorf("web = %+v", w)
|
||||||
|
}
|
||||||
|
if len(m.Snapshot().Wants) != 0 {
|
||||||
|
t.Errorf("a hold was taken for a want: %+v", m.Snapshot().Wants)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1906/N23-AC2: given a hold ask answered "keep" (the ask ended with it), when the operator releases the hold, then the view
|
||||||
|
// writes its answer "release" to that asking under the ask's view key, though the ask was answered; the fingerprint may be the
|
||||||
|
// start of its hex. The same answer again, an unknown hold and another option are refused, and write nothing.
|
||||||
|
func TestAKeptHoldCanBeReleased(t *testing.T) {
|
||||||
|
m, w := New(), &memWriter{}
|
||||||
|
m.SetWriter(w)
|
||||||
|
e, a := holdAsk(t)
|
||||||
|
fill(m, e, heldE(t, "plex", fpNew, "ace", "health failed"), answered(t, e.Key, a.ID, "keep"))
|
||||||
|
short := strings.TrimPrefix(fpNew, delivery.SourcePrefix2)[:8]
|
||||||
|
if _, err := m.AnswerHold("plex", short, HoldRelease, "it was a flake"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
va, err := delivery.DecodeViewAnswer(w.puts[e.Key+".view"])
|
||||||
|
if err != nil || va.Asking != a.ID || va.Option != HoldRelease || va.Words != "it was a flake" {
|
||||||
|
t.Fatalf("the view's answer = %+v, %v", va, err)
|
||||||
|
}
|
||||||
|
w.puts = nil
|
||||||
|
fill(m, answered(t, e.Key, a.ID, HoldRelease))
|
||||||
|
for name, try := range map[string]func() error{
|
||||||
|
"again": func() error { _, err := m.AnswerHold("plex", fpNew, HoldRelease, ""); return err },
|
||||||
|
"unknown": func() error { _, err := m.AnswerHold("plex", "ffff", HoldKeep, ""); return err },
|
||||||
|
"option": func() error { _, err := m.AnswerHold("plex", fpNew, "restore", ""); return err },
|
||||||
|
"module": func() error { _, err := m.AnswerHold("web", fpNew, HoldKeep, ""); return err },
|
||||||
|
} {
|
||||||
|
if err := try(); err == nil {
|
||||||
|
t.Errorf("%s: not refused", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(w.puts) != 0 {
|
||||||
|
t.Errorf("a refused answer wrote %v", w.puts)
|
||||||
|
}
|
||||||
|
// And kept again: the operator may change their mind both ways.
|
||||||
|
if _, err := m.AnswerHold("plex", fpNew, HoldKeep, ""); err != nil {
|
||||||
|
t.Errorf("keep after release: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -311,7 +311,10 @@ func (r Reader) BuilderBuild(module, fp2, commit string) (BuilderRow, error) {
|
|||||||
return BuilderRow{}, fmt.Errorf("the indexer's value of %s is not readable: %w", module, err)
|
return BuilderRow{}, fmt.Errorf("the indexer's value of %s is not readable: %w", module, err)
|
||||||
}
|
}
|
||||||
if commit != "" && m.Commit != commit {
|
if commit != "" && m.Commit != commit {
|
||||||
return BuilderRow{}, fmt.Errorf("the indexer holds %s at commit %s, not %s, so its fingerprint is not the one of that commit", module, m.Commit, commit)
|
// The value names the commit its fingerprint first appeared at, and a later merge that left the module alone
|
||||||
|
// writes none (the operator, 2026-10-25): another commit may well have the same fingerprint, which is not guessed.
|
||||||
|
return BuilderRow{}, fmt.Errorf("the indexer holds %s at commit %s, the commit its fingerprint first appeared at, not %s; "+
|
||||||
|
"a later trunk commit that left the module alone keeps that fingerprint: ask without commit for the build at it", module, m.Commit, commit)
|
||||||
}
|
}
|
||||||
if m.FP2 == "" {
|
if m.FP2 == "" {
|
||||||
return BuilderRow{}, fmt.Errorf("the indexer holds no new-form fingerprint of %s: %s", module, m.None2)
|
return BuilderRow{}, fmt.Errorf("the indexer holds no new-form fingerprint of %s: %s", module, m.None2)
|
||||||
|
|||||||
@@ -46,6 +46,8 @@ type ask struct {
|
|||||||
Key string `json:"key,omitempty"`
|
Key string `json:"key,omitempty"`
|
||||||
Option string `json:"option,omitempty"`
|
Option string `json:"option,omitempty"`
|
||||||
Words string `json:"words,omitempty"`
|
Words string `json:"words,omitempty"`
|
||||||
|
Module string `json:"module,omitempty"`
|
||||||
|
FP string `json:"fp,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ServeSocket answers {"ask":"rows"} on l, line-delimited JSON, until ctx ends, and {"ask":"answer","key":…,"option":…,"words":…}
|
// ServeSocket answers {"ask":"rows"} on l, line-delimited JSON, until ctx ends, and {"ask":"answer","key":…,"option":…,"words":…}
|
||||||
@@ -72,8 +74,21 @@ func answer(c net.Conn, m *Model) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
var a ask
|
var a ask
|
||||||
if json.Unmarshal(line, &a) != nil || (a.Ask != "rows" && a.Ask != "answer") {
|
if json.Unmarshal(line, &a) != nil || (a.Ask != "rows" && a.Ask != "answer" && a.Ask != "holds" && a.Ask != "hold") {
|
||||||
json.NewEncoder(c).Encode(map[string]string{"error": `refused: this socket answers {"ask":"rows"} and {"ask":"answer"} and nothing else`})
|
json.NewEncoder(c).Encode(map[string]string{"error": `refused: this socket answers {"ask":"rows"}, {"ask":"answer"}, {"ask":"holds"} and {"ask":"hold"} and nothing else`})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch a.Ask {
|
||||||
|
case "holds":
|
||||||
|
json.NewEncoder(c).Encode(m.Holds())
|
||||||
|
return
|
||||||
|
case "hold":
|
||||||
|
r, err := m.AnswerHold(a.Module, a.FP, a.Option, a.Words)
|
||||||
|
if err != nil {
|
||||||
|
json.NewEncoder(c).Encode(map[string]string{"error": err.Error()})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
json.NewEncoder(c).Encode(r)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if a.Ask == "answer" {
|
if a.Ask == "answer" {
|
||||||
|
|||||||
@@ -231,8 +231,10 @@ type Model struct {
|
|||||||
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
|
chosen map[string]delivery.ViewAnswer // the answers the view wrote, by ask key
|
||||||
writer Writer
|
writer Writer
|
||||||
backlog *Backlog
|
backlog *Backlog
|
||||||
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
|
answers map[string]string // by key: the router id of the asking the teller's answer is for ("" when unreadable)
|
||||||
newest map[string]string // module -> id
|
answerOf map[string]delivery.Answer // by key: the teller's answer whole, when readable (holds.go)
|
||||||
|
holds map[string]heldEntry // the assigner's holds, by held key (holds.go)
|
||||||
|
newest map[string]string // module -> id
|
||||||
haves map[string]haveEntry
|
haves map[string]haveEntry
|
||||||
nodes map[string]nodeEntry
|
nodes map[string]nodeEntry
|
||||||
// the watchman's buckets, by key
|
// the watchman's buckets, by key
|
||||||
@@ -344,7 +346,7 @@ type nodeEntry struct {
|
|||||||
|
|
||||||
// New is an empty model.
|
// New is an empty model.
|
||||||
func New() *Model {
|
func New() *Model {
|
||||||
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, newest: map[string]string{},
|
return &Model{assigns: map[string]assignment{}, builds: map[string]buildEntry{}, groups: map[string]groupEntry{}, modules: map[string]moduleEntry{}, keys: map[string]keyEntry{}, asks: map[string]askRecord{}, wants: map[string]wantEntry{}, told: map[string]bool{}, initial: map[string]bool{}, composed: map[string]composedEntry{}, aasks: map[string]assignerAskRecord{}, chosen: map[string]delivery.ViewAnswer{}, answers: map[string]string{}, answerOf: map[string]delivery.Answer{}, holds: map[string]heldEntry{}, newest: map[string]string{},
|
||||||
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
|
haves: map[string]haveEntry{}, nodes: map[string]nodeEntry{},
|
||||||
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}}
|
presence: map[string]presenceEntry{}, stuck: map[string]stuckEntry{}}
|
||||||
}
|
}
|
||||||
@@ -486,9 +488,12 @@ func (m *Model) apply(e Entry) {
|
|||||||
m.chosen[askKey] = a
|
m.chosen[askKey] = a
|
||||||
}
|
}
|
||||||
case delivery.AssignerWantsBucket:
|
case delivery.AssignerWantsBucket:
|
||||||
|
if m.applyHeld(e) {
|
||||||
|
return
|
||||||
|
}
|
||||||
node, module, err := delivery.ParseWantKey(e.Key)
|
node, module, err := delivery.ParseWantKey(e.Key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return // a hold, and the node's own key, are not a module's want
|
return // the node's own key is not a module's want
|
||||||
}
|
}
|
||||||
key, kerr := delivery.HaveKey(node, module)
|
key, kerr := delivery.HaveKey(node, module)
|
||||||
if kerr != nil {
|
if kerr != nil {
|
||||||
@@ -520,6 +525,7 @@ func (m *Model) apply(e Entry) {
|
|||||||
case delivery.AnswersBucket:
|
case delivery.AnswersBucket:
|
||||||
if e.Deleted {
|
if e.Deleted {
|
||||||
delete(m.answers, e.Key)
|
delete(m.answers, e.Key)
|
||||||
|
delete(m.answerOf, e.Key)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// The answer's warrant is `<router id>@<time>`: it ends that asking, and only that one (a later asking of the key is
|
// The answer's warrant is `<router id>@<time>`: it ends that asking, and only that one (a later asking of the key is
|
||||||
@@ -527,6 +533,9 @@ func (m *Model) apply(e Entry) {
|
|||||||
id := ""
|
id := ""
|
||||||
if a, err := delivery.DecodeAnswer(e.Value); err == nil {
|
if a, err := delivery.DecodeAnswer(e.Value); err == nil {
|
||||||
id, _, _ = strings.Cut(a.Warrant, "@")
|
id, _, _ = strings.Cut(a.Warrant, "@")
|
||||||
|
m.answerOf[e.Key] = a
|
||||||
|
} else {
|
||||||
|
delete(m.answerOf, e.Key)
|
||||||
}
|
}
|
||||||
m.answers[e.Key] = id
|
m.answers[e.Key] = id
|
||||||
case delivery.BuilderBuildsBucket:
|
case delivery.BuilderBuildsBucket:
|
||||||
|
|||||||
Reference in New Issue
Block a user