Compare commits

..
711 changed files with 4292 additions and 72323 deletions
-3
View File
@@ -1,4 +1 @@
vendor/
# A compiled program never enters a commit (issue 1577): `go build .` in a command directory leaves one there.
modules/*/cmd/*/mesh-*
!modules/*/cmd/*/mesh-*.go
-7
View File
@@ -1,7 +0,0 @@
# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with
# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry
# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's
# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here
# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it.
mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after
mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after
@@ -119,11 +119,11 @@ func (p *passer) composeAtHead(ctx context.Context, m assign.Mesh, ch assign.Cha
if h := ch.Members[repository]; h != "" {
commits, member = []string{h}, true
} else {
commits = p.w.runningCommits(module)
commits = p.w.runningCommits(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module)
}
if len(commits) == 0 {
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: the builder's log holds no done "+
"trunk build of it, and the group holds no member of "+repository)
notJudged = append(notJudged, "not judged whether "+module+" reads the group's manifests: no build of it is assigned with a "+
"commit on record, and the group holds no member of "+repository)
continue
}
for _, commit := range commits {
@@ -106,7 +106,7 @@ func TestAGroupWithAnSDKMemberIsComposedByTheComposerAtItsHead(t *testing.T) {
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
running := strings.Repeat("3", 40)
b.putJSON(t, provider.AssignmentsBucket, "novox.mesh-controller", []byte(`{"node":"novox","module":"mesh-controller","build":"c-1"}`))
b.built(t, "mesh-controller", "novox/mesh-controller", running)
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("mesh-controller", "c-1"), []byte(`{"id":"c-1","repository":"novox/mesh-controller","commit":"`+running+`"}`))
sdkHead := strings.Repeat("4", 40)
heads := []delivery.Head{{Repository: "novox/mesh-sdk", Number: 80, Head: sdkHead}, {Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
hash, _ := delivery.HeadsHash(heads)
@@ -7,10 +7,8 @@ import (
"encoding/json"
"fmt"
"io"
"maps"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
@@ -19,12 +17,9 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose/judge"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provider"
@@ -74,21 +69,12 @@ func startBed(t *testing.T, leaveOut string) *bed {
if name == leaveOut {
continue
}
cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
if name == delivery.PlacementPulsesBucket {
cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
}
kv, err := js.CreateKeyValue(ctx, cfg)
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
if err != nil {
t.Fatal(err)
}
b.buckets[name] = kv
}
// The builder's module log, as the runtime raises it (ADR 0297): the running commits are read from it (ADR 0571 clause 1).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog),
Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
t.Fatal(err)
}
@@ -142,7 +128,9 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) {
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
b := startBed(t, "")
done := b.start(t)
events, err := b.js.Stream(b.ctx, "EVENTS")
if err != nil {
t.Fatal(err)
@@ -167,7 +155,7 @@ func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) {
if err != nil {
t.Fatal(err)
}
m, err := catalogue.Decode(raw)
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
@@ -184,9 +172,6 @@ func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) {
// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket.
func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
for _, missing := range readBuckets {
if startsWithout(missing) {
continue // it starts without it (the placement and the grants' source: main_test.go; the puts: below)
}
b := startBed(t, missing)
select {
case err := <-b.start(t):
@@ -233,34 +218,6 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
t.Fatal("the put was not taken further once its bucket was made")
}
}
// Without the seat's placement state it starts all the same, and once the bucket is made (the controller makes
// it from the manifest) it is read with no restart: a module only the placement names is
// said as placed by one source only.
pb := startBed(t, delivery.PlacementBucket)
pdone := pb.start(t)
select {
case err := <-pdone:
t.Fatalf("without %s the assigner stopped: %v", delivery.PlacementBucket, err)
case <-time.After(2 * time.Second):
}
placement, err := pb.js.CreateKeyValue(pb.ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: 5})
if err != nil {
t.Fatal(err)
}
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "web"}}})
plkey, _ := delivery.LegacyPlacementKey("n1") // the form before, read beside the new (research 072 row 37)
if _, err := placement.Put(pb.ctx, plkey, pl); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(15 * time.Second); ; time.Sleep(200 * time.Millisecond) {
if c, ok := pb.conditionsSaid(t)[delivery.AssignerModule+".placement.n1.web"]; ok && c.State == asks.StateRaised {
break
}
if time.Now().After(deadline) {
t.Fatal("the placement state made after the start was not read")
}
}
}
// putJSON writes a value into one of the bed's buckets.
@@ -271,19 +228,6 @@ func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
}
}
// built appends the end of a done trunk build of module at commit to the builder's log, as the builder does.
func (b *bed) built(t *testing.T, module, repository, commit string) {
t.Helper()
raw, err := delivery.EncodeBuildLogEntry(delivery.BuildLogEntry{Event: delivery.BuildEndedEvent, Module: module, FP: delivery.SourcePrefix2 + strings.Repeat("c", 64),
Repository: repository, Directory: delivery.RootPath, Commit: commit, Node: "novox", At: time.Now().UTC(), Outcome: delivery.BuildDone})
if err != nil {
t.Fatal(err)
}
if _, err := b.js.Publish(b.ctx, buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog)+"."+module, raw); err != nil {
t.Fatal(err)
}
}
// compose says the controller composed module for node at fp, with the digest d (a letter repeated) and the judgement of the
// move to it, in mesh-controller_composed.
func (b *bed) compose(t *testing.T, node, module, fp string, d byte, judgement delivery.Judgement) string {
@@ -462,32 +406,6 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if !cleared {
t.Errorf("the condition of web on neither list on g14 was not cleared once g14 got its want")
}
// Issue 1126 (ADR 0547 §2): the catalogue's placement reaches the indexer's state naming web on every node the
// assignments name it on. Each node then composes its own: its want names the build alone (Self, no composition), and
// its own want says it composes its own, which the node agent reads (mesh-host's TestNats...ComposesAndAppliesItsOwn).
for _, node := range []string{"ace", "g14", "shanks"} {
pl, err := delivery.EncodePlacement(delivery.Placement{Node: node, Modules: []delivery.PlacedModule{{Module: "web"}}})
if err != nil {
t.Fatal(err)
}
pkey, _ := delivery.LegacyPlacementKey(node)
b.putJSON(t, delivery.PlacementBucket, pkey, pl)
}
if w := b.wantNow(t, "ace", "web", "ace's want once placed: the build alone, composed by the node", func(w delivery.Want) bool { return w.Self }); w.Composed != "" || w.FP != fNew {
t.Errorf("ace's want once placed = %+v", w)
}
nkey, _ := delivery.WantNodeKey("ace")
var self bool
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && !self; time.Sleep(100 * time.Millisecond) {
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, nkey); err == nil {
nw, err := delivery.DecodeNodeWant(e.Value())
self = err == nil && nw.Self
}
}
if !self {
t.Error("ace's own want does not say it composes its own once placed")
}
}
// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
@@ -1025,231 +943,3 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
}
}
}
// bedStore is the runtime's states over the bed's buckets, as the seat's verbs read and write them: the assigner's own by
// their local names, another module's `<module>.<state>`.
type bedStore struct{ b *bed }
func (s bedStore) kv(state string) jetstream.KeyValue {
if strings.Contains(state, ".") {
return s.b.buckets[strings.Replace(state, ".", "_", 1)]
}
return s.b.buckets[delivery.AssignerModule+"_"+state]
}
func (s bedStore) get(state, key string) ([]byte, uint64, bool, error) {
e, err := s.kv(state).Get(s.b.ctx, key)
if err != nil {
return nil, 0, false, nil
}
return e.Value(), e.Revision(), true, nil
}
func (s bedStore) put(state, key string, value []byte) error {
_, err := s.kv(state).Put(s.b.ctx, key, value)
return err
}
func (s bedStore) keys(state string) ([]string, error) {
keys, err := s.kv(state).Keys(s.b.ctx)
if err != nil && strings.Contains(err.Error(), "no keys") {
return nil, nil
}
return keys, err
}
func (s bedStore) del(state, key string) error { return s.kv(state).Delete(s.b.ctx, key) }
func (s bedStore) history(string, string) ([]stdio.StateRevision, error) { return nil, nil }
// 1126 (ADR 0567 §2-§6), the seat mesh-placement whole on the bus: a proposal moving web from the anchor to the laptop is
// proposed through the verb, the assigner finds both nodes must answer, they answer the same in their haves, the
// operator is asked and approves, and each key is written at a new revision while placement-changed is heard on the
// assigner's stream naming both nodes. A second proposal laid over the old revision of a key moved since is laid again,
// and declined, writes nothing and is withdrawn.
func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
b := startBed(t, "")
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "MOD_" + delivery.AssignerModule + "_placement",
Subjects: []string{process.ModuleStreamSubject(delivery.AssignerModule, ">")}}); err != nil {
t.Fatal(err)
}
heard := make(chan delivery.PlacementEvent, 16)
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
if _, err := nc.Subscribe(process.ModuleStreamSubject(delivery.AssignerModule, delivery.PlacementChangedEvent), func(m *nats.Msg) {
var e delivery.PlacementEvent
if json.Unmarshal(m.Data, &e) == nil {
heard <- e
}
}); err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}})
b.putJSON(t, delivery.HaveBucket, node+".node", rec)
}
b.putJSON(t, delivery.ModulesBucket, "web", []byte(`{"v":1}`))
b.putJSON(t, delivery.ModulesBucket, "shell", []byte(`{"v":1}`))
// place writes a node's placement as another writer would, in its node and module keys (research 072 row 37), and
// answers its node key's revision; legacy writes it in the form before, placement.<node>, alone.
place := func(node string, legacy bool, modules ...string) uint64 {
p := delivery.Placement{Node: node, Modules: []delivery.PlacedModule{}}
for _, m := range modules {
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
}
values, _ := delivery.PlacementValues(p)
key, _ := delivery.PlacementNodeKey(node)
if legacy {
key, _ = delivery.LegacyPlacementKey(node)
values = map[string][]byte{key: values[key]}
}
for _, k := range slices.Sorted(maps.Keys(values)) {
if k == key {
continue
}
if _, err := b.buckets[delivery.PlacementBucket].Put(b.ctx, k, values[k]); err != nil {
t.Fatal(err)
}
}
rev, err := b.buckets[delivery.PlacementBucket].Put(b.ctx, key, values[key])
if err != nil {
t.Fatal(err)
}
return rev
}
place("anchor", true, "web")
laptopRev := place("laptop", false, "shell")
bound := judge.AnswerBound
t.Cleanup(func() { judge.AnswerBound = bound })
judge.AnswerBound = 2 * time.Second
saysWhatItKeeps(t, b, b.start(t))
// The holder converts anchor, held in the form before alone, to its node and module keys once at its start.
var anchorRev uint64
for deadline := time.Now().Add(15 * time.Second); anchorRev == 0; time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "anchor.node"); err == nil {
anchorRev = e.Revision()
if _, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, "anchor.web"); err != nil {
t.Fatalf("anchor converted without its module key: %v", err)
}
}
if time.Now().After(deadline) {
t.Fatal("anchor in the form before was not converted")
}
}
verbs := newSeatTools(bedStore{b}, func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"ok":true}`), nil })
propose := func() delivery.PlacementProposal {
t.Helper()
got, err := verbs.doPropose(map[string]any{"why": "move web", "changes": `[{"op":"take-off","node":"anchor","module":"web"},` +
`{"op":"place","node":"laptop","module":"web"}]`}, "operator@laptop")
if err != nil {
t.Fatal(err)
}
return got.(proposed).Proposal
}
proposalNow := func(id, what string, ok func(delivery.PlacementProposal) bool) delivery.PlacementProposal {
t.Helper()
var last delivery.PlacementProposal
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.PlacementProposalsBucket].Get(b.ctx, id); err == nil {
if p, err := delivery.DecodePlacementProposal(e.Value()); err == nil {
if last = p; ok(p) {
return p
}
}
}
}
t.Fatalf("the proposal %s never was %s: %+v", id, what, last)
return last
}
answerAll := func(p delivery.PlacementProposal) {
t.Helper()
for _, node := range p.Must {
key, _ := delivery.PlacementCandidateKey(p.ID, node)
e, err := b.buckets[delivery.PlacementCandidatesBucket].Get(b.ctx, key)
if err != nil {
t.Fatalf("no candidate %s: %v", key, err)
}
have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}, Candidates: map[string]delivery.CandidateAnswer{
p.ID: {Proposal: p.ID, Revision: e.Revision(), Verdict: delivery.CandidateSame, Same: []string{"node"}}}})
b.putJSON(t, delivery.HaveBucket, node+".node", have)
}
}
p := propose()
p = proposalNow(p.ID, "told who must answer", func(p delivery.PlacementProposal) bool { return len(p.Must) > 0 })
if !slices.Equal(p.Must, []string{"anchor", "laptop"}) {
t.Fatalf("must answer = %v, want both nodes", p.Must)
}
answerAll(p)
p = proposalNow(p.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
akey, _ := delivery.PlacementAskKey(p.ID)
b.answer(t, akey, assign.OptionYes)
select {
case e := <-heard:
if e.Proposal.ID != p.ID || len(e.Nodes) != 2 || e.Nodes[0].Before != anchorRev || e.Nodes[1].Before != laptopRev ||
e.Nodes[0].After <= anchorRev || e.Warrant == nil || !slices.Equal(e.Nodes[0].Removed, []string{"web"}) {
t.Errorf("placement-changed = %+v", e)
}
case <-time.After(15 * time.Second):
t.Fatal("no placement-changed heard")
}
for node, want := range map[string][]string{"anchor": {}, "laptop": {"shell", "web"}} {
// The node key names the modules, each module key carries its own, and the mirror of the form before follows.
values := map[string][]byte{}
for _, key := range []string{node + ".node", "placement." + node, node + ".web", node + ".shell"} {
if e, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, key); err == nil {
values[key] = e.Value()
}
}
got, err := delivery.PlacementsOf(values)
if err != nil || !slices.Equal(got[node].Assigned(), want) {
t.Errorf("%s = %v, %v, want %v", node, got[node].Assigned(), err, want)
}
mirror, _ := delivery.DecodePlacement(values["placement."+node])
if !slices.Equal(mirror.Assigned(), want) {
t.Errorf("the mirror placement.%s = %v, want %v", node, mirror.Assigned(), want)
}
}
if b.hasKey(delivery.PlacementProposalsBucket, p.ID) {
t.Error("an applied proposal stays in the state")
}
// A proposal whose key moves before its warrant is laid again; then declined, it writes nothing and ends.
q, err := verbs.doPropose(map[string]any{"why": "back", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
if err != nil {
t.Fatal(err)
}
moved := q.(proposed).Proposal
place("laptop", false, "shell", "web") // another writer moved the key
relaid := proposalNow(moved.ID, "laid again", func(p delivery.PlacementProposal) bool {
return strings.Contains(p.Words, "laid again") && len(p.Must) > 0
})
answerAll(relaid)
proposalNow(moved.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
akey, _ = delivery.PlacementAskKey(moved.ID)
b.answer(t, akey, assign.OptionNo)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.PlacementProposalsBucket, moved.ID); time.Sleep(50 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("a declined proposal was not ended")
}
}
key, _ := delivery.PlacementNodeKey("laptop")
if e, _ := b.buckets[delivery.PlacementBucket].Get(b.ctx, key); e == nil || !strings.Contains(string(e.Value()), "web") {
t.Error("a declined proposal wrote its key")
}
// ADR 0567 §4: a node that does not answer fails the verdict once its pulse key's time to live ends, heard on the bus
// with no timer of the assigner's, naming it.
q, err = verbs.doPropose(map[string]any{"why": "again", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
if err != nil {
t.Fatal(err)
}
silent := q.(proposed).Proposal
failed := proposalNow(silent.ID, "failed for a silent node", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalFailed })
if failed.Verdict == nil || !strings.Contains(strings.Join(failed.Verdict.Fails, "; "), "laptop (offline) did not answer") {
t.Errorf("the silent node's verdict = %+v", failed.Verdict)
}
}
@@ -14,30 +14,23 @@ import (
"log"
"os"
"os/signal"
"slices"
"sync/atomic"
"strings"
"syscall"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provider"
"git.novox.be/novox/mesh-sdk/go/provisions"
"mesh-delivery-assigner/internal/assign"
)
func main() {
if os.Getenv(process.BrokerFileEnv) == "" {
if os.Getenv("MESH_BROKER_FILE") == "" {
// Started by the tool runner, with no credential of its own: the put's tools (ADR 0546), served through the runtime,
// since a process never serves (ADR 0436). The process the manifest runs names its credential.
if err := serveTools(); err != nil {
@@ -57,11 +50,9 @@ func main() {
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
// holds the two lists equal), so the controller's broker grants exactly these.
var readBuckets = []string{
buscompose.ResolvedBucket,
provider.AssignmentsBucket,
delivery.BuildsBucket,
delivery.ControllerGrantsBucket,
delivery.PartGrantsBucket,
buscompose.LeaseBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
@@ -71,52 +62,25 @@ var readBuckets = []string{
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.StepsBucket(goBuildModule),
takeoverBucket,
provisions.ProvisionsBucket,
keptBucket,
}
// readLogs are the logs of other parts the assigner reads, each named in its manifest's reads: the builder's log, whose
// newest done trunk build of a module names the commit a reader of the manifests runs at (novox/hq ADR 0571 clause 1).
var readLogs = []string{delivery.BuildLogRead}
// takeoverBucket is the condition keeper's take-over key's bucket (novox/hq ADR 0564 §9): whether the assigner's said
// conditions go straight to the router or only as findings (issue 1952 WM6).
var takeoverBucket = health.Bucket(health.KeeperModule, health.TakeoverState)
// goBuildModule is the Go build module, which takes the assigner's compose asks on its seat's check stream and writes
// their steps in its steps state (novox/hq ADR 0482 clause 4).
const goBuildModule = "mesh-build-go"
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
// (ADR 0567 §1), named after the assigner as every module state is.
// ownBuckets are the assigner's own states, as its manifest declares them.
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket,
delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket}
// seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants
// them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind).
var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposalsBucket, delivery.PlacementCandidatesBucket,
delivery.PlacementPulsesBucket}
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by
// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3).
// and the teller, its puts by whoever follows one (ADR 0546).
var keptStates = []announce.State{
{Name: delivery.PlacementState, Readable: true},
{Name: delivery.PlacementProposalsState, Readable: true},
{Name: delivery.PlacementCandidatesState, Readable: true},
{Name: delivery.PlacementPulsesState, Readable: true},
{Name: "asks", Readable: true},
{Name: "compositions", Readable: true},
{Name: "differences", Readable: true},
{Name: "puts", Readable: true},
{Name: "wants", Readable: true},
{Name: health.FindingsState, Readable: true},
{Name: delivery.AssignerPoliciesState, Readable: true},
{Name: delivery.SeatHoldersState, Readable: true},
}
// buckets is every bucket the assigner opened, by name.
@@ -128,78 +92,12 @@ type buckets map[string]jetstream.KeyValue
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
out := buckets{}
// The controller's lease first (ADR 0549 §5, ADR 0561 §3): once it is handed away the seat's placement alone names
// where modules run, and the controller's assignments are no bucket the assigner needs. Not granted or not there, the
// controller holds it, as before.
away, why := false, ""
if kv, err := js.KeyValue(ctx, buscompose.LeaseBucket); err != nil {
logf("the bucket %s cannot be opened (%v): the controller is taken to hold its lease, so the assignments are read beside the placement", buscompose.LeaseBucket, err)
} else {
out[buscompose.LeaseBucket] = kv
e, gerr := kv.Get(ctx, buscompose.ControllerLeaseKey)
var raw []byte
if gerr == nil {
raw = e.Value()
}
if errors.Is(gerr, jetstream.ErrKeyNotFound) {
gerr = nil
}
away, why = buscompose.LeaseHandedAway(raw, raw != nil, gerr)
}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
if name == buscompose.LeaseBucket {
continue
}
kv, err := js.KeyValue(ctx, name)
if name == provider.AssignmentsBucket && err != nil && away {
logf("the bucket %s cannot be opened (%v), and none is needed: %s, so the seat's placement alone is read", name, err, why)
continue
}
if name == delivery.AssignerPutsBucket && err != nil {
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
if slices.Contains(seatBuckets, name) && err != nil {
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement the controller's
// assignments alone name where modules run, as before any placement was written (ADR 0549 §5).
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == delivery.SeatHoldersBucket && err != nil {
// The seat's state, new with ADR 0571 §4: bound once the bus says it was made; until then nobody is told who
// holds which seat here, and the controller's seats still answer.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == keptBucket && err != nil {
// mesh-config's held values (ADR 0565 §5): without them a composition lays none, said in the journal.
logf("the bucket %s cannot be opened (%v): no held value is laid on a composition until it is bound", name, err)
continue
}
if name == provisions.ProvisionsBucket && err != nil {
// The seat mesh-provisioner's state, with its own delivery (ADR 0570 §4): bound once the bus says it was made;
// until then no want waits for a provision, as before.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if name == assign.PoliciesBucket && err != nil {
// The seat mesh-wants' state, new with it (ADR 0565 §7): bound once the bus says it was made; until then no
// person's policy is read, so the controller's beside each composition holds, as before.
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if (name == assign.FindingsBucket || name == takeoverBucket) && err != nil {
// New with health leaving the controller (issue 1952 WM6): without them the conditions are said straight to the
// router and kept as no finding, as before.
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
continue
}
if sourceOfGrants(name) && err != nil {
// The grants part's grants and the controller's lease choose where grants are read from (issue 774): without
// them the controller's grants are read, as before the lease is handed away.
logf("the bucket %s cannot be opened (%v): the controller's grants are read until the assigner starts again", name, err)
continue
}
if err != nil {
return nil, process.EssentialErr("assigner", name, "write a want from the whole mesh", err)
}
@@ -208,35 +106,36 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
return out, nil
}
// startsWithout says the assigner starts without the bucket when it cannot be opened (open says why for each): the seat's
// states and those new with a delivery of their own, bound once made, and the two that choose the grants' source.
func startsWithout(name string) bool {
switch name {
case delivery.SeatHoldersBucket, keptBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket, takeoverBucket,
delivery.AssignerPutsBucket:
return true
}
return slices.Contains(seatBuckets, name) || sourceOfGrants(name)
}
// sourceOfGrants says the bucket is one of the two that choose the grants' source, the grants part's grants and the
// controller's lease: neither stops the assigner when it is not there (delivery.GrantSource).
func sourceOfGrants(name string) bool {
return name == delivery.PartGrantsBucket || name == buscompose.LeaseBucket
}
// run is the program: it returns when ctx ends or the bus cannot be used.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
account, err := process.ReadBrokerAccount(getenv)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
}
account, err := process.ReadAccount(path)
if err != nil {
return err
}
conn, js, err := account.Dial()
opts := []nats.Option{
nats.Name(account.Node + "." + account.Module),
nats.UserInfo(account.User, account.Password),
nats.CustomInboxPrefix(account.InboxPrefix()),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
}
if strings.TrimSpace(account.Fingerprint) != "" {
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
}
conn, err := nats.Connect(account.URL, opts...)
if err != nil {
return err
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
kvs, err := open(ctx, js, logger.Printf)
if err != nil {
return err
@@ -251,78 +150,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := said.Keeps(keptStates...); err != nil {
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
}
thisNode = account.Node
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
findings := findingsOf(ctx, kvs, logger.Printf)
buildLog := func(handle func(stdio.LogEntry) error) error {
return said.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Watch("", stdio.LogFromAll, handle)
}
return follow(ctx, js, conn, kvs, findings.Say(sayVia(ctx, js)), findings, emitting{said}, buildLog, logger)
return follow(ctx, js, kvs, sayVia(ctx, js), emitting{said}, logger)
}
// findingsOf is where the assigner's said conditions go as findings (issue 1952 WM6): its findings bucket, and the
// condition keeper's take-over key, watched, so a condition is said straight to the router only until the take-over. A
// bucket it cannot open is said: without its findings the conditions are said straight, as before; without the take-over
// key they are said straight too, the controller being followed.
func findingsOf(ctx context.Context, kvs buckets, logf func(string, ...any)) assign.Findings {
var f assign.Findings
if kv := kvs[assign.FindingsBucket]; kv != nil {
f.Put = func(key string, value []byte) error { _, err := kv.Put(ctx, key, value); return err }
f.Delete = func(key string) error { return kv.Delete(ctx, key) }
f.Keys = func() ([]string, error) {
keys, err := kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return keys, err
}
}
var keeper atomic.Bool
f.ToKeeper = keeper.Load
takeover := takeoverBucket
kv := kvs[takeover]
if kv == nil {
return f
}
w, err := kv.Watch(ctx, health.TakeoverKey)
if err != nil {
process.NotWatched(logf, takeover, err, "the controller is followed: the conditions are said straight to the router")
return f
}
go func() {
defer w.Stop()
for {
select {
case <-ctx.Done():
return
case e, ok := <-w.Updates():
if !ok {
logf("the watch of %s ended: the last source it named is followed until the assigner starts again", takeover)
return
}
if e == nil {
continue
}
var raw []byte
if e.Operation() == jetstream.KeyValuePut {
raw = e.Value()
}
t, err := health.DecodeTakeover(raw, raw != nil)
if err != nil {
logf("the take-over key cannot be read (%v): the controller is followed", err)
}
if was := keeper.Swap(t.Follow == health.FollowKeeper); was != keeper.Load() {
logf("the assigner's conditions are said by %s from now (the take-over key follows %s)", t.Seat(), t.Follow)
}
}
}
}()
return f
}
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
var thisNode string
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
type emitting struct{ *process.Conn }
@@ -331,35 +162,24 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
var watched = []string{
buscompose.ResolvedBucket,
provider.AssignmentsBucket,
delivery.ControllerGrantsBucket,
delivery.PartGrantsBucket,
buscompose.LeaseBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ModulesBucket,
delivery.PlacementBucket,
delivery.PlacementProposalsBucket,
delivery.PlacementCandidatesBucket,
delivery.PlacementPulsesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.AssignerAsksBucket,
delivery.AssignerCompositionsBucket,
delivery.AssignerDifferencesBucket,
delivery.AssignerWantsBucket,
assign.PoliciesBucket,
provisions.ProvisionsBucket,
keptBucket,
delivery.SeatHoldersBucket,
}
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, kvs buckets, say func(asks.Condition) error, findings assign.Findings, bus caller, buildLog logWatch, logger *log.Logger) error {
func follow(ctx context.Context, js jetstream.JetStream, kvs buckets, say func(asks.Condition) error, bus caller, logger *log.Logger) error {
w := newWorld()
dirty := make(chan struct{}, 1)
wake := func() {
@@ -368,22 +188,6 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
default:
}
}
// The builder's log (ADR 0571 clause 1): every entry it keeps before the first pass, then each new one as it is appended;
// a module's newer done trunk build makes a pass, as a composition waiting on its commit is judged then. Where it cannot be
// read, a composition names no running reader of the manifests until the assigner starts again, and it is said.
if buildLog != nil {
var first atomic.Bool
if err := buildLog(func(le stdio.LogEntry) error {
if w.putBuilt(le.Value) && first.Load() {
wake()
}
return nil
}); err != nil {
logger.Printf("%s is not read (%v): no composition is judged against a running reader of the manifests until the assigner starts again",
delivery.BuildLogRead, err)
}
first.Store(true)
}
errs := make(chan error, len(watched)+2)
settled := make(chan struct{}, len(watched)+2)
watch := func(name string, opts ...jetstream.WatchOpt) error {
@@ -419,13 +223,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if e.Operation() == jetstream.KeyValuePut {
value = e.Value()
}
before := w.grantsBucket()
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
if name == buscompose.LeaseBucket && w.grantsBucket() == before {
// The controller renews its lease every few seconds: only a change of where grants are read from
// makes a pass (delivery.GrantSource).
continue
}
w.put(name, e.Key(), value)
wake()
}
}
@@ -461,16 +259,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
return
}
if bus != nil {
p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway)
p.acts = newPutActs(bus, wake, logger.Printf)
}
logger.Printf("the bucket %s is open: puts are taken further", delivery.AssignerPutsBucket)
}
started := 0
for _, name := range watched {
if kvs[name] == nil && (startsWithout(name) || name == provider.AssignmentsBucket) {
continue // not on the bus at this start (open said so)
}
started++
var opts []jetstream.WatchOpt
if name == delivery.HaveBucket {
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
@@ -482,37 +275,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
return err
}
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then the
// controller's assignments alone name where modules run (ADR 0549 §5).
type appeared struct {
name string
kv jetstream.KeyValue
}
placementAppeared := make(chan appeared, len(seatBuckets)+2)
for _, name := range seatBuckets {
if kvs[name] == nil && js != nil {
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it",
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
}
}
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket, assign.PoliciesBucket, provisions.ProvisionsBucket, keptBucket, delivery.SeatHoldersBucket} {
if kvs[name] == nil && js != nil {
name := name
process.AwaitBucket(ctx, js, nc, name, "until then the bucket is not read",
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
}
}
takePlacement := func(a appeared) error {
if kvs[a.name] != nil {
return nil // bound already, by the other of its two events
}
kvs[a.name] = a.kv
logger.Printf("the bucket %s is open and watched", a.name)
return watch(a.name)
}
for range started + 1 {
for range len(watched) + 1 {
select {
case <-ctx.Done():
return nil
@@ -521,16 +284,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case <-settled:
}
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
p.acts = newPutActs(bus, wake, logger.Printf, w.handedAway)
}
if bus != nil {
p.seat = newSeatActs(bus, wake, logger.Printf, w.handedAway)
}
if js != nil {
p.publish = streamPublisher(ctx, js, thisNode)
p.acts = newPutActs(bus, wake, logger.Printf)
}
wake()
for {
@@ -540,29 +297,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
case err := <-errs:
return err
case <-settled:
case kv := <-placementAppeared:
if err := takePlacement(kv); err != nil {
return err
}
case <-dirty:
reopenPuts(p)
// The grants part makes the seat's buckets in the pass that seals the list granting them, so their creation can
// be said before this credential may hear it: every pass, itself started by a watched change, binds each one the
// bus now holds, and says the seat waits while one is missing.
var unbound []string
for _, name := range seatBuckets {
if kvs[name] != nil || js == nil {
continue
}
if kv, err := js.KeyValue(ctx, name); err == nil {
if err := takePlacement(appeared{name, kv}); err != nil {
return err
}
continue
}
unbound = append(unbound, name)
}
p.sayWaits(unbound)
p.pass(ctx)
}
}
@@ -1,8 +1,6 @@
package main
import (
"mesh-delivery-assigner/internal/assign"
"context"
"errors"
"fmt"
@@ -19,39 +17,25 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
func TestOpensWhatItsManifestNames(t *testing.T) {
// 1297/B2a: without a credential the program does not start, and says why.
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
t.Errorf("run without a credential = %v", err)
}
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
}
m, err := catalogue.Decode(raw)
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
var reads []string
for _, r := range m.Reads {
if slices.Contains(readLogs, r) {
continue // a log, read through its own reader (ADR 0571 clause 1), not a bucket
}
reads = append(reads, strings.Replace(r, ".", "_", 1))
}
for _, l := range readLogs {
if !slices.Contains(m.Reads, l) {
t.Errorf("the program reads the log %s, the manifest does not name it", l)
}
}
var states []string
for _, s := range m.State {
states = append(states, m.Module+"_"+s.Name)
@@ -74,30 +58,20 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
// Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the
// assignments refused while the controller holds it stop the start, and once it is handed away they are not needed.
handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)
for _, c := range []struct {
missing string
err error
starts bool
lease []byte
said string
}{
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""},
// Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start.
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"},
{buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"},
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""},
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"},
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
@@ -120,7 +94,7 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
kvs[delivery.ControllerComposedBucket].(*watchKV).entries = []kvEntry{{key: ckey, value: cd}}
kvs[delivery.AssignerPutsBucket].(*watchKV).closed = true
kvs[delivery.AssignerWantsBucket].(*watchKV).wrote = wrote
go follow(ctx, nil, nil, kvs, func(asks.Condition) error { return nil }, assign.Findings{}, nil, nil, log.New(io.Discard, "", 0))
go follow(ctx, nil, kvs, func(asks.Condition) error { return nil }, nil, log.New(io.Discard, "", 0))
select {
case key := <-wrote:
if key != "want.ace.node" {
@@ -131,23 +105,26 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
}
// 1297/B2a: without a credential the program does not start, and says why.
func TestNoCredential(t *testing.T) {
err := run(context.Background(), func(string) string { return "" }, io.Discard)
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
t.Errorf("run without a credential = %v", err)
}
}
// refusingJS opens every bucket but one, which it refuses with err.
type refusingJS struct {
jetstream.JetStream
missing string
err error
lease []byte // the controller's lease, when it holds one
}
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
if name == r.missing {
return nil, r.err
}
kv := &memKV{values: map[string][]byte{}}
if name == buscompose.LeaseBucket && r.lease != nil {
kv.values[buscompose.ControllerLeaseKey] = r.lease
}
return kv, nil
return &memKV{values: map[string][]byte{}}, nil
}
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
@@ -193,5 +170,3 @@ type kvEntry struct {
func (e kvEntry) Key() string { return e.key }
func (e kvEntry) Value() []byte { return e.value }
func (e kvEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
func (e kvEntry) Revision() uint64 { return 1 }
func (e kvEntry) Created() time.Time { return time.Time{} }
@@ -1,11 +1,9 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"maps"
"fmt"
"slices"
"strings"
"time"
@@ -13,7 +11,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-sdk/go/sdkversion"
"mesh-delivery-assigner/internal/assign"
@@ -30,16 +27,6 @@ type passer struct {
// pass says a condition once and a restart says it again (the router keeps one per key).
say func(asks.Condition) error
said map[string]bool
// findings is the assigner's findings bucket (its said conditions and policy-split, ADR 0564 §2, ADR 0565 §8); splits the
// modules whose policy-split finding stands, nil until the first pass, which deletes any a previous process left.
findings assign.Findings
splits map[string]bool
// failedProvisions are the failed provisions whose finding this process wrote (ADR 0570 §4).
failedProvisions map[string]bool
// missing are the required provisions with no key this process pulsed, and missingSaid those whose finding it wrote.
missing, missingSaid map[string]bool
// unplaced are the nodes whose unplaced finding this process wrote.
unplaced map[string]bool
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
answeredAt map[string]uint64
asks *composeAsks
@@ -53,27 +40,6 @@ type passer struct {
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
noWant map[string]string
// unreached is, by node and module, the words last said why a wanted module may not be reached there (issue 1960: its
// provider is down, which holds no want): said once per words, and once more when it is reached again.
unreached map[string]string
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
// §1, issue 1496): raised once per words, cleared once when it ends.
nodeFailed map[string]string
// callSaid is, by the key of a refused controller call's condition, what is raised (issue 774): by this process, or by
// a previous one and taken over from the findings.
callSaid map[string]*callSaid
// callsSwept says the first pass took over the refused calls' conditions a previous process raised (sweepCalls).
callsSwept bool
// The seat mesh-placement's holder (seat.go, ADR 0567): its calls of the controller (nil: none is made), how it says
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
seat *seatActs
publish publisher
saidProposed map[string]bool
// convertedAll says every node in the form before was converted to its node and module keys (convertPlacement).
convertedAll bool
// saidWaits is the last said of the seat's wait for its states (sayWaits).
saidWaits string
intended map[string]map[string]map[string]bool
}
func (p *passer) clock() time.Time {
@@ -84,7 +50,6 @@ func (p *passer) clock() time.Time {
}
func (p *passer) pass(ctx context.Context) {
p.placementSeat(ctx)
p.compositions(ctx)
p.wants(ctx)
p.puts(ctx)
@@ -96,23 +61,7 @@ func (p *passer) pass(ctx context.Context) {
// controller has composed it and again whenever that digest changes (ADR 0539 §6).
func (p *passer) nodeWants(ctx context.Context) {
written := p.w.read(delivery.AssignerWantsBucket)
placed := p.w.placedNodes()
// A node the catalogue places composes its own composition (ADR 0547 §2): its own want names none.
for node := range placed {
key, err := delivery.WantNodeKey(node)
if err != nil {
continue
}
if before, err := delivery.DecodeNodeWant(written[key]); err == nil && before.Self {
continue
}
want := delivery.NodeWant{Self: true, Words: "the node composes its own composition (ADR 0547 §2)"}
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeNodeWant(want) })
}
for node, d := range p.w.nodeComposed() {
if placed[node] {
continue
}
key, err := delivery.WantNodeKey(node)
if err != nil {
continue
@@ -149,17 +98,6 @@ func (p *passer) wants(ctx context.Context) {
}
}
wanted := p.w.wantsOf()
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
provisionsOf := p.provisionsOf()
missing := map[string]bool{}
defer func() { p.provisionsMissing(ctx, missing) }()
defer p.unplacedFindings()
defer p.seatHolders(ctx)
defer p.policySplits(manifests, chosen)
differ := p.withCalls(p.w.placementDiffers())
p.sayCalls(differ)
frozen := p.sayPlacement(differ)
placed := p.w.placedNodes()
puts := p.putsByModule()
p.staleAsks(ctx, trunk)
names := map[string]bool{}
@@ -176,29 +114,15 @@ func (p *passer) wants(ctx context.Context) {
// quiet at the router.
first := p.neither == nil
stale := map[string]neitherOne{}
left := map[string]string{}
defer func() { p.sayNeither(neither, stale, left) }()
// Where grants are read from, once for the pass: the grants part's while the lease is handed away (issue 774).
grants := p.w.grantsBucket()
defer func() { p.sayNeither(neither, stale) }()
for name := range names {
fp := trunk[name].FP2
name := name
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
Self: func(node string) bool { return placed[node] },
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
if placed[node] && p.w.handedAway() {
return p.w.ownComposed(node) // from the lease write: the digest its own have names (ADR 0571 §6)
}
if placed[node] {
// The node composes its own (ADR 0547 §2, §4): no composition of the controller's is waited for, the want
// names the build alone, and the node judges the move itself.
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither, Words: "the node composes its own"}, true
}
return p.w.composed(node, name, fp)
},
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) },
GrantedAt: func(fp string) bool {
key, err := delivery.ControllerGrantKey(name, fp)
return err == nil && p.w.has(grants, key)
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
},
Restored: func(node string) bool {
key, err := delivery.RestoreAskKey(node, name, fp)
@@ -214,8 +138,8 @@ func (p *passer) wants(ctx context.Context) {
hold, _ := delivery.HoldAskKey(name, fp)
m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
if raw := p.w.value(grants, grant); raw != nil {
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
}
@@ -229,17 +153,8 @@ func (p *passer) wants(ctx context.Context) {
last[node] = p.w.lastHealthy(node, name, fp)
}
out := assign.Wants(m, nodes, wanted[name], last)
for _, node := range m.Nodes {
for _, prov := range m.MissingProvisions(node) {
missing[node+"."+name+"."+prov] = true
}
}
p.yeses(ctx, name, m, wanted[name], &out)
gatherNeither(neither, name, out)
leftNeither(left, m, wanted[name], out)
p.sayNoWant(name, m.Nodes, wanted[name], out)
p.sayUnreached(name, m.Nodes, out)
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
if first {
startClears(stale, m, out)
if fp != "" && !m.Held && out.Held == nil && !out.Lifted {
@@ -267,9 +182,6 @@ func (p *passer) wants(ctx context.Context) {
p.recordAsk(ctx, name, fp, need)
}
for node, want := range out.Wants {
if frozen[node+"."+name] {
continue // one source alone names it (ADR 0549 §5): no want, and the one written before stays
}
key, err := delivery.WantKey(node, name)
if err != nil {
p.logf("no want of %s on %s is written: %v", name, node, err)
@@ -298,7 +210,7 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
}
reason := ""
if h, held := out.Rules[node]; held && (!written || w.Removed) {
reason = h.Reason()
reason = fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
last, said := p.noWant[key]
switch {
@@ -317,80 +229,6 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
}
}
// sayUnreached says in the journal, once per words, that a module is wanted on a node and may not be reached there, since
// no provider of what it requires runs healthy (rule 5, which holds no want: issue 1960), and once when it is reached again.
func (p *passer) sayUnreached(module string, nodes []string, out assign.Outcome) {
if p.unreached == nil {
p.unreached = map[string]string{}
}
for _, node := range nodes {
key := node + "." + module
why, last := out.Unreached[node], p.unreached[key]
switch {
case why != "" && why != last:
p.logf("%s on %s is wanted and may not be reached: %s", module, node, why)
p.unreached[key] = why
case why == "" && last != "":
p.logf("%s on %s is reached: its providers run healthy", module, node)
delete(p.unreached, key)
}
}
}
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
// after a start remembers nothing a previous process raised: it clears, for every module, the condition of every node the
// presence bucket knows that does not fail alone, since a node that recovered while no process watched leaves no failed have to
// find it by (a clear of a condition not open is quiet at the router). A condition the router did not take is said again at the next pass.
func (p *passer) sayNodeFailed(module, fp string, nodes []string, known assign.Nodes, out assign.Outcome, first bool) {
if p.nodeFailed == nil {
p.nodeFailed = map[string]string{}
}
look := map[string]bool{}
for _, node := range nodes {
look[node] = true
}
for key := range p.nodeFailed {
if node, m, _ := strings.Cut(key, "."); m == module {
look[node] = true
}
}
if first {
for node := range known.Online {
look[node] = true
}
}
for _, node := range slices.Sorted(maps.Keys(look)) {
key := node + "." + module
f, failing := out.NodeFailed[node]
last, said := p.nodeFailed[key]
switch {
case failing && f.Words+"|"+strings.Join(f.Others, ",") != last:
p.logf("%s on %s: the health check at %s failed here (%s), while it runs healthy on %s: the failure is %s's, so nothing "+
"is held or set back", module, node, fp, f.Words, strings.Join(f.Others, ", "), node)
if p.say != nil {
if err := p.say(assign.NodeFailedCondition(node, module, fp, f, true)); err != nil {
p.logf("the condition of %s failing on %s alone could not be said, and is said at the next pass: %v", module, node, err)
continue
}
}
p.nodeFailed[key] = f.Words + "|" + strings.Join(f.Others, ",")
case !failing && (said || first):
if said {
p.logf("%s on %s no longer fails its health check alone", module, node)
}
if p.say != nil {
if err := p.say(assign.NodeFailedCondition(node, module, fp, assign.NodeFailure{}, false)); err != nil && said {
p.logf("the condition of %s failing on %s alone could not be cleared, and is cleared at the next pass: %v", module, node, err)
continue
}
}
delete(p.nodeFailed, key)
}
}
}
// neitherOne is a module assigned to a node that holds no want for it after a pass, and the rule holding its first want back.
type neitherOne struct {
node, module string
@@ -404,48 +242,6 @@ func gatherNeither(into map[string]neitherOne, module string, out assign.Outcome
}
}
// leftNeither says, by node and module, for every node of m not on neither list after the pass, why: "" when the node has a live
// want for it (written before and kept, or written in this pass), else why it has none. sayNeither clears a neither condition in
// these words, so a clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while the bus bar
// left it with none).
func leftNeither(into map[string]string, m assign.Module, before map[string]delivery.Want, out assign.Outcome) {
for _, node := range m.Nodes {
if _, on := out.Neither[node]; on {
continue
}
w, written := out.Wants[node]
if !written {
w, written = before[node]
}
why := ""
h, ruled := out.Rules[node]
switch {
case written && !w.Removed:
case (m.Held && !out.Lifted) || out.Held != nil:
why = "it is held, and its held condition says so"
case slices.ContainsFunc(out.Record, func(r assign.RecordNeed) bool { return r.Node == node }):
why = "its move waits for the operator's yes on its record ask"
case putOn(m, node):
why = "its put waits on its own steps and the operator's answer"
case m.FP == "":
why = "the trunk has no build of it"
case ruled:
why = h.Reason()
default:
why = "the node does not run it"
}
into[node+"."+m.Name] = why
}
}
// putOn says whether an operator's put on node owns m's want there: one still on its steps (ADR 0546), so the rules write none
// meanwhile. An ended put (done, refused, released) stays in the bucket and owns nothing.
func putOn(m assign.Module, node string) bool {
p, put := m.Puts[node]
_, owns := assign.PutOwns(p)
return put && owns
}
// startClears adds, for the first pass after a start, every node of m that is not on neither list after the pass: a neither
// condition a previous process raised for it is cleared. That covers a node with a want and one with none that is not on neither
// list, such as a held module (issue 1945: the hold is said on its own, so a neither raised for it before the restart would stay up
@@ -462,9 +258,7 @@ func startClears(into map[string]neitherOne, m assign.Module, out assign.Outcome
// once, the condition of each one said before that has a want again (issue 1945). A say that fails is made again at the next pass.
// stale is, on the first pass after a start, the modules whose condition a previous process may have raised and that have a want:
// each is cleared once.
// left is, by node and module, why each assigned node not on neither list after this pass has no want ("" when it has one); a key
// it does not hold is no longer assigned.
func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]string) {
func (p *passer) sayNeither(now, stale map[string]neitherOne) {
if p.neither == nil {
p.neither = map[string]neitherOne{}
for key, one := range stale {
@@ -484,29 +278,21 @@ func (p *passer) sayNeither(now, stale map[string]neitherOne, left map[string]st
continue
}
}
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; what holds its first want back: %s",
one.module, one.node, one.holding.Reason())
p.logf("%s is assigned to %s and on neither list: no want, and not in the node's own composition; rule %d holds its first want back: %s",
one.module, one.node, one.holding.Rule, one.holding.Words)
p.neither[key] = one
}
for key, one := range p.neither {
if _, still := now[key]; still {
continue
}
why, assigned := left[key]
if !assigned {
why = "it is no longer assigned there"
}
if p.say != nil {
if err := p.say(assign.NeitherLeftCondition(one.node, one.module, why)); err != nil {
if err := p.say(assign.NeitherCondition(one.node, one.module, assign.Holding{}, false)); err != nil {
p.logf("the condition of %s on neither list on %s could not be cleared, and is cleared at the next pass: %v", one.module, one.node, err)
continue
}
}
if why == "" {
p.logf("%s on %s has a want again", one.module, one.node)
} else {
p.logf("%s on %s is no longer on neither list and has no want: %s", one.module, one.node, why)
}
p.logf("%s on %s has a want again", one.module, one.node)
delete(p.neither, key)
}
}
@@ -569,11 +355,6 @@ func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]b
p.logf("%s %s is not written: %v", bucket, key, err)
return
}
// A value the bucket already holds is not put again: every put is a change its watchers hear, and a want put unchanged
// made its node compose again (issue 774, 2026-10-31: novox composed every 8 s, two asks of the vault each time).
if held := p.w.value(bucket, key); held != nil && bytes.Equal(held, value) {
return
}
if _, err := p.kvs[bucket].Put(ctx, key, value); err != nil && !errors.Is(err, context.Canceled) {
p.logf("%s %s could not be written: %v", bucket, key, err)
return
@@ -653,14 +434,6 @@ func (p *passer) staleAsk(key string, trunk map[string]delivery.Module) bool {
if !on {
return true
}
// A node that composes its own asks about its own composition, which its have names as the one it waits on (yeses): that
// ask stands while the have waits on it, whatever the controller composes (issue 774, 2026-10-31: novox's asks of
// postgres, keycloak and mongodb were taken away and made again under a new id every pass, so no answer could count).
if hkey, err := delivery.HaveKey(k.Node, k.Module); err == nil {
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, hkey)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits == k.Composed {
return false
}
}
// The assigner asks about a composition at more than the trunk fingerprint: a node held back by the rules is asked about the
// revised composition of the fingerprint it runs (ADR 0545), and a node's first want at the older fingerprint it runs. So a
// record ask is stale only when its digest is the composition of none of the fingerprints the node can still want (the
@@ -902,360 +675,3 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
}
p.said[c.Key] = true
}
// sayPlacement raises, once, the condition of every pair the controller's assignments and the catalogue's placement do not
// both name, clears it once they agree, and answers those pairs by node and module (ADR 0549 §5): the pass writes no want
// for them and keeps the one written before. A say that fails is made again at the next pass.
func (p *passer) sayPlacement(differ []assign.Pair) map[string]bool {
frozen := map[string]bool{}
now := map[string]bool{}
for _, d := range differ {
frozen[d.Node+"."+d.Module] = true
c := assign.PlacementCondition(d, true)
now[c.Key] = true
if p.said[c.Key+".raised"] {
continue
}
p.logf("%s on %s is named by %s alone: no want is written for it there until both sources name it (ADR 0549 §5)", d.Module, d.Node, d.Only)
if err := p.say(c); err != nil {
p.logf("the placement condition of %s on %s could not be said, and is said at the next pass: %v", d.Module, d.Node, err)
continue
}
p.said[c.Key+".raised"] = true
}
prefix := delivery.AssignerModule + ".placement."
for key := range p.said {
c, raised := strings.CutSuffix(key, ".raised")
if !raised || !strings.HasPrefix(c, prefix) || now[c] {
continue
}
node, module, _ := strings.Cut(strings.TrimPrefix(c, prefix), ".")
if err := p.say(assign.PlacementCondition(assign.Pair{Node: node, Module: module}, false)); err != nil {
p.logf("the placement condition of %s on %s could not be cleared, and is cleared at the next pass: %v", module, node, err)
continue
}
delete(p.said, key)
}
return frozen
}
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
// person must allow (ADR 0547 §3): for a have of phase
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again.
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) {
for _, node := range m.Nodes {
if m.Self == nil || !m.Self(node) {
continue
}
was, had := before[node]
w, written := out.Wants[node]
if !written {
if !had || was.Removed {
continue
}
w = was
}
if w.Removed {
continue
}
yes := ""
if had && was.FP == w.FP {
yes = was.Yes
}
key, err := delivery.HaveKey(node, module)
if err != nil {
continue
}
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
if m.Allowed != nil && m.Allowed(node, h.Waits) {
yes = h.Waits
} else {
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
}
}
// A want carried over from before the node composed its own may still name the controller's composition; a placed
// node's want names the build alone (ADR 0547 §2).
if yes != w.Yes || w.Composed != "" || !w.Self {
w.Yes, w.Composed, w.Self = yes, "", true
out.Wants[node] = w
}
}
}
// chosenOf is a person's choice for module, nil when none.
func chosenOf(chosen map[string]catalogue.PersonPolicy, module string) *catalogue.PersonPolicy {
if c, ok := chosen[module]; ok {
return &c
}
return nil
}
// policySplits writes the finding policy-split for each module whose policy at the assigner differs from one the controller
// writes beside a composition of it, and deletes it once they agree (ADR 0565 §8): written when a split begins, never again
// while it stands. The first pass after a start deletes every one a previous process left that no longer splits.
func (p *passer) policySplits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) {
if p.findings.Put == nil {
return
}
now := time.Now
if p.now != nil {
now = p.now
}
split := map[string]bool{}
theirs := map[string][]string{}
if !p.w.handedAway() { // the controller writes no policy from the lease write (ADR 0571 §6)
theirs = assign.ControllerPolicies(p.w.read(delivery.ControllerComposedBucket))
}
for _, s := range assign.Splits(manifests, chosen, theirs) {
split[s.Module] = true
if p.splits[s.Module] {
continue
}
value, err := assign.PolicySplitFinding(s, now())
if err == nil {
err = p.findings.Put(assign.PolicySplitKey(s.Module), value)
}
if err != nil {
p.logf("the policy split of %s is not written: %v", s.Module, err)
delete(split, s.Module)
}
}
gone := p.splits
if gone == nil && p.findings.Keys != nil {
gone = map[string]bool{}
if keys, err := p.findings.Keys(); err == nil {
for _, k := range keys {
if m, ok := strings.CutSuffix(strings.TrimPrefix(k, "module."), "."+assign.KindPolicySplit); ok && strings.HasPrefix(k, "module.") {
gone[m] = true
}
}
}
}
for m := range gone {
if !split[m] {
if err := p.findings.Delete(assign.PolicySplitKey(m)); err != nil {
p.logf("the policy split of %s is not deleted: %v", m, err)
split[m] = true // tried again at the next pass
}
}
}
p.splits = split
}
// provisionsOf answers, per module, the provisions it consumes on a node, by key (ADR 0570 §4), from the seat
// mesh-provisioner's state; nil while the state is not read (the seat not delivered yet), so no want waits on it. A failed
// provision is also said as the assigner's finding `provider.<key>.provision-failed` (a waiting one is the watchman's
// needs-operator), deleted once it is no longer failed.
func (p *passer) provisionsOf() func(module string) func(node string) map[string]provisions.Provision {
if p.kvs[provisions.ProvisionsBucket] == nil {
return func(string) func(string) map[string]provisions.Provision { return nil }
}
by := map[string]map[string]map[string]provisions.Provision{} // module, node, key
failed := map[string]provisions.Provision{}
for key, raw := range p.w.read(provisions.ProvisionsBucket) {
pr, err := provisions.Decode(raw)
if err != nil {
continue
}
node, module := pr.Consumer.Node, pr.Consumer.Module
if by[module] == nil {
by[module] = map[string]map[string]provisions.Provision{}
}
if by[module][node] == nil {
by[module][node] = map[string]provisions.Provision{}
}
by[module][node][key] = pr
if pr.State == provisions.Failed {
failed[key] = pr
}
}
p.provisionFindings(failed)
return func(module string) func(string) map[string]provisions.Provision {
return func(node string) map[string]provisions.Provision { return by[module][node] }
}
}
// provisionFindings writes the finding of each failed provision and deletes the ones no longer failed.
func (p *passer) provisionFindings(failed map[string]provisions.Provision) {
if p.findings.Put == nil {
return
}
now := time.Now
if p.now != nil {
now = p.now
}
for key, pr := range failed {
if p.failedProvisions[key] {
continue
}
value, err := assign.ProvisionFailedFinding(key, pr, now())
if err == nil {
err = p.findings.Put(assign.ProvisionFailedKey(key), value)
}
if err != nil {
p.logf("the failed provision %s is not said: %v", key, err)
continue
}
if p.failedProvisions == nil {
p.failedProvisions = map[string]bool{}
}
p.failedProvisions[key] = true
}
for key := range p.failedProvisions {
if _, still := failed[key]; still {
continue
}
if err := p.findings.Delete(assign.ProvisionFailedKey(key)); err != nil {
p.logf("the failed provision %s is not cleared: %v", key, err)
continue
}
delete(p.failedProvisions, key)
}
}
// MissingBound is how long a required provision may have no key before it is said: a module's delivery bound, the default
// when its manifest names none (ADR 0564 clause 13 (d)).
var MissingBound = delivery.DefaultDeliveryBound
// missingPulse is the pulse key of a provision with no key, `provision-missing.<node>.<module>.<provision>`.
func missingPulse(key string) string { return "provision-missing." + key }
// provisionsMissing is the loud end of a required provision with no key (the lead, 2026-10-29, ADR 0570 §4): each one is
// pulsed with the bound when it is first seen; once its pulse expired and it still has no key, the finding
// `provider.<node>.<module>.<provision>.provision-missing` is written; one keyed again ends both. The pulse's expiry, heard on
// the watch, is the pass that writes it: no timer.
func (p *passer) provisionsMissing(ctx context.Context, missing map[string]bool) {
pulses := p.kvs[delivery.PlacementPulsesBucket]
if p.missing == nil {
p.missing = map[string]bool{}
}
for key := range missing {
switch {
case !p.missing[key]:
p.missing[key] = true
if pulses != nil {
p.pulse(ctx, missingPulse(key), MissingBound)
}
case pulses == nil || !p.pulsing(missingPulse(key)):
if p.missingSaid[key] || p.findings.Put == nil {
continue
}
value, err := assign.ProvisionMissingFinding(key, MissingBound, p.clock())
if err == nil {
err = p.findings.Put(assign.ProvisionMissingKey(key), value)
}
if err != nil {
p.logf("the missing provision %s is not said: %v", key, err)
continue
}
if p.missingSaid == nil {
p.missingSaid = map[string]bool{}
}
p.missingSaid[key] = true
}
}
for key := range p.missing {
if missing[key] {
continue
}
delete(p.missing, key)
if pulses != nil && p.pulsing(missingPulse(key)) {
if err := pulses.Delete(ctx, missingPulse(key)); err == nil {
p.w.put(delivery.PlacementPulsesBucket, missingPulse(key), nil)
}
}
if p.missingSaid[key] && p.findings.Delete != nil {
if err := p.findings.Delete(assign.ProvisionMissingKey(key)); err != nil {
p.logf("the missing provision %s is not cleared: %v", key, err)
p.missing[key] = true
continue
}
delete(p.missingSaid, key)
}
}
}
// unplacedFindings writes, from the lease write, the finding `machine.<node>.unplaced` for each node that runs but no
// placement names, which gets no want (ADR 0571 §6), and deletes it once the node is placed.
func (p *passer) unplacedFindings() {
if p.findings.Put == nil {
return
}
now := []string(p.w.unplacedNodes())
is := map[string]bool{}
for _, node := range now {
is[node] = true
if p.unplaced[node] {
continue
}
value, err := assign.UnplacedFinding(node, p.clock())
if err == nil {
err = p.findings.Put(assign.UnplacedKey(node), value)
}
if err != nil {
p.logf("the unplaced node %s is not said: %v", node, err)
continue
}
if p.unplaced == nil {
p.unplaced = map[string]bool{}
}
p.unplaced[node] = true
}
for node := range p.unplaced {
if is[node] {
continue
}
if err := p.findings.Delete(assign.UnplacedKey(node)); err == nil {
delete(p.unplaced, node)
}
}
}
// seatHolders writes the seat mesh-placement's state seat-holders from every node's placement (ADR 0571 §4): one key per
// seat held once, `<seat>`, and one per node of a node seat, `<seat>.<node>`, written only where it differs, and a key no
// placement holds any more deleted. A pass follows each placement change and each manifest the indexer takes in, so no
// timer writes it.
func (p *passer) seatHolders(ctx context.Context) {
kv := p.kvs[delivery.SeatHoldersBucket]
if kv == nil {
return
}
placements := p.w.placements()
if len(placements) == 0 {
return // no placement read yet: nothing is known of who holds what, and nothing is deleted
}
want := delivery.SeatHoldersOf(placements)
have := p.w.read(delivery.SeatHoldersBucket)
for key, sh := range want {
value, err := json.Marshal(sh)
if err != nil || string(have[key]) == string(value) {
continue
}
p.put(ctx, delivery.SeatHoldersBucket, key, func() ([]byte, error) { return value, nil })
}
for key := range have {
if _, ok := want[key]; ok {
continue
}
if err := kv.Delete(ctx, key); err != nil {
p.logf("the seat holders %s are not deleted: %v", key, err)
continue
}
p.w.put(delivery.SeatHoldersBucket, key, nil)
}
}
// answering is, by provision, the manifests of the modules placed on some node that provide it: what a want's
// provisions are judged by (catalogue.Awaited, issue 774). A provider in the catalogue placed nowhere answers nobody.
func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(string) []catalogue.Manifest {
return func(provision string) []catalogue.Manifest {
var out []catalogue.Manifest
for _, name := range slices.Sorted(maps.Keys(n.Providers[provision])) {
if m, ok := manifests[name]; ok && len(n.Providers[provision][name]) > 0 {
out = append(out, m)
}
}
return out
}
}
@@ -1,21 +1,18 @@
package main
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
@@ -27,16 +24,9 @@ import (
type memKV struct {
jetstream.KeyValue
values map[string][]byte
puts int // how many puts reached the bucket
}
func (m *memKV) Put(_ context.Context, key string, value []byte) (uint64, error) {
m.values[key] = value
m.puts++
return uint64(len(m.values)), nil
}
func (m *memKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
m.values[key] = value
return uint64(len(m.values)), nil
}
@@ -145,9 +135,6 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin
type memStore map[string]*memKV
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
if s[state] == nil {
return nil, 0, false, nil
}
v, ok := s[state].values[key]
return v, uint64(len(v)), ok, nil
}
@@ -196,7 +183,7 @@ func putOfTheBus(t *testing.T) {
p := newPasser(kvs, now)
bus := &fakeBus{calls: make(chan string, 8)}
woke := make(chan struct{}, 8) // each act's goroutine wakes a pass once it is done
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, nil)
p.acts = newPutActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {})
p.acts.snapshot = func(context.Context, string, string) (string, error) {
return "novox's restore point of nats (streams) taken 22:01", nil
}
@@ -297,7 +284,7 @@ func putOfTheBus(t *testing.T) {
if got.Step != delivery.PutWanted || err != nil || w.FP != fp || w.Composed != digest || !w.Put || !w.Hold {
t.Fatalf("snapshotted, the put stands at %s (%s); the want %+v, %v", got.Step, got.Words, w, err)
}
if call := <-bus.calls; !strings.HasPrefix(call, "seat:mesh-hand-acts.warranted ") || !strings.Contains(call, a.Ask.ID) { // ADR 0565 §11: into the seat
if call := <-bus.calls; !strings.Contains(call, "warranted") || !strings.Contains(call, a.Ask.ID) {
t.Errorf("the warrant was recorded as %q", call)
}
world(delivery.AssignerWantsBucket, "want.novox.nats", wantsKV.values["want.novox.nats"])
@@ -355,9 +342,6 @@ func putOfTheBus(t *testing.T) {
// again after a say that failed, and clears it once when the module has a want again; a module on neither list is also written to
// the journal, so it is loud with no operator channel.
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t)
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
var said []asks.Condition
failing := true
@@ -373,66 +357,30 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
p.logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
one := map[string]neitherOne{"g14.screen-lock": {node: "g14", module: "screen-lock",
holding: assign.Holding{Rule: delivery.RuleProviders, Words: "no provider of x11-display runs healthy on g14 or at its canary"}}}
p.sayNeither(one, nil, nil) // the say fails
p.sayNeither(one, nil, nil)
p.sayNeither(one, nil, nil)
p.sayNeither(one, nil) // the say fails
p.sayNeither(one, nil)
p.sayNeither(one, nil)
if len(said) != 1 || said[0].State != asks.StateRaised || said[0].Subject.Machine != "g14" || said[0].Subject.ID != "screen-lock" {
t.Fatalf("said = %+v, want one raised condition of screen-lock on g14", said)
}
if !strings.Contains(strings.Join(logged, "\n"), "screen-lock") {
t.Errorf("the journal does not name the module on neither list: %q", logged)
}
wants := map[string]string{"g14.screen-lock": ""}
p.sayNeither(map[string]neitherOne{}, nil, wants)
p.sayNeither(map[string]neitherOne{}, nil, wants)
p.sayNeither(map[string]neitherOne{}, nil)
p.sayNeither(map[string]neitherOne{}, nil)
if len(said) != 2 || said[1].State != asks.StateCleared || said[1].Key != said[0].Key {
t.Errorf("said = %+v, want the condition cleared once under its key", said)
}
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
t.Errorf("the clear said %q", last)
}
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
p.sayNeither(bus, nil, nil)
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
t.Errorf("the bus's neither condition = %+v", c)
}
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
if c := said[len(said)-1]; c.State != asks.StateCleared || strings.Contains(c.Resolved, "has a want") {
t.Errorf("a module that left neither list with no want was cleared as %+v", c)
}
if last := logged[len(logged)-1]; last != "nats on novox is no longer on neither list and has no want: it is held, and its held condition says so" {
t.Errorf("the clear without a want said %q", last)
}
}
// 1945/AC21: the pass gathers the modules on neither list of every module's outcome, keyed by node and module.
func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
func TestThePassGathersTheModulesOnNeitherList(t *testing.T) {
got := map[string]neitherOne{}
gatherNeither(got, "screen-lock", assign.Outcome{Neither: map[string]assign.Holding{"g14": {Rule: delivery.RuleGrant, Words: "w"}}})
gatherNeither(got, "rofi", assign.Outcome{})
if len(got) != 1 || got["g14.screen-lock"].node != "g14" || got["g14.screen-lock"].module != "screen-lock" || got["g14.screen-lock"].holding.Rule != delivery.RuleGrant {
t.Errorf("gathered = %+v", got)
}
// 1945 (the bus bar): a node off neither list is cleared in words that say why it has no want; an operator's put running
// through its steps owns the want, so it is named, never "the node does not run it".
for _, c := range []struct {
name string
puts map[string]delivery.Put
before map[string]delivery.Want
why string
}{
{"a live want", nil, map[string]delivery.Want{"novox": {FP: fpTest}}, ""},
{"a put on its steps", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutPerson}}, nil, "its put waits on its own steps and the operator's answer"},
{"nothing", nil, nil, "the node does not run it"},
{"an ended put", map[string]delivery.Put{"novox": {FP: fpTest, Step: delivery.PutRefused}}, nil, "the node does not run it"},
} {
left := map[string]string{}
leftNeither(left, assign.Module{Name: "nats", FP: fpTest, Nodes: []string{"novox"}, Puts: c.puts}, c.before, assign.Outcome{})
if why, there := left["novox.nats"]; !there || why != c.why {
t.Errorf("%s: left %q (%v), want %q", c.name, why, there, c.why)
}
}
// 1945 (a first want held back silently): the journal says why a node gets no want when the reason appears or changes, and
// when it has a want again: one line per change, not per pass.
p := newPasser(buckets{}, time.Date(2026, 10, 25, 21, 30, 0, 0, time.UTC))
@@ -451,7 +399,6 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
{nil, wanted, "rofi on shanks has a want"},
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
} {
logged = nil
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
@@ -459,71 +406,12 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
t.Errorf("pass %d said %q, want %q", i, got, c.line)
}
}
// 1496, ADR 0557 §1: a failure that is the node's own (its fingerprint healthy elsewhere, not held) raises the node's condition
// once, naming the node, the module, F and the have's words, and clears it once when it ends.
var conds []asks.Condition
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
failing := map[string]assign.NodeFailure{"shanks": {Others: []string{"ace"}, Words: "health failed, phase running: w"}}
for i, c := range []struct {
failed map[string]assign.NodeFailure
state string // the condition said in this pass, "" for none
}{
{failing, asks.StateRaised},
{failing, ""},
{nil, asks.StateCleared},
{nil, ""},
} {
conds = nil
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: c.failed}, false)
switch {
case c.state == "" && len(conds) != 0, c.state != "" && (len(conds) != 1 || conds[0].State != c.state):
t.Errorf("node failed, pass %d said %+v, want %q", i, conds, c.state)
case c.state == asks.StateRaised && (conds[0].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" ||
!strings.Contains(conds[0].Explanation, "failed on shanks (health failed, phase running: w), while it runs healthy on ace")):
t.Errorf("node failed, raised %+v", conds[0])
}
}
// D2: a node that leaves the module's nodes while it failed alone is cleared once; the first pass after a start clears every
// node the presence bucket knows, for every module, failed have or not (a node that recovered unwatched leaves none).
conds = nil
p.sayNodeFailed("rofi", fpTest, []string{"shanks"}, assign.Nodes{}, assign.Outcome{NodeFailed: failing}, false)
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
p.sayNodeFailed("rofi", fpTest, nil, assign.Nodes{}, assign.Outcome{}, false)
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != "mesh-delivery-assigner.failed-alone.shanks.rofi" {
t.Errorf("a node leaving the module said %+v, want one raise and one clear", conds)
}
known := assign.Nodes{Online: map[string]bool{"ace": true, "g14": false},
Haves: map[string]map[string]delivery.Have{"g14": {"rofi": {FP: fpTest, Health: delivery.HealthFailed}}}}
for _, c := range []struct {
name string
known assign.Nodes
keys []string
}{
{"a failed have", known, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
{"no failed have", assign.Nodes{Online: known.Online}, []string{"mesh-delivery-assigner.failed-alone.ace.rofi", "mesh-delivery-assigner.failed-alone.g14.rofi"}},
} {
conds = nil
fresh := newPasser(buckets{}, time.Date(2026, 10, 26, 22, 0, 0, 0, time.UTC))
fresh.logf = func(string, ...any) {}
fresh.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
fresh.sayNodeFailed("rofi", fpTest, nil, c.known, assign.Outcome{}, true)
var keys []string
for _, cond := range conds {
if cond.State == asks.StateCleared {
keys = append(keys, cond.Key)
}
}
if !slices.Equal(keys, c.keys) || len(conds) != len(c.keys) {
t.Errorf("first pass, %s: said %+v, want clears of %q", c.name, conds, c.keys)
}
}
}
// A neither or held condition a previous process raised is cleared by the first pass after a start: a fresh pass remembers
// nothing it said, so it clears, once, the neither condition of every assigned module not on neither list and the held condition of
// every module that is not held. Clearing a condition that is not open is quiet at the router.
func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
t.Run("a held module", testAHeldModuleRaisesNoNeitherAndClearsAStandingOne)
wantsKV := &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerWantsBucket: wantsKV, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
var said []asks.Condition
@@ -561,7 +449,7 @@ func TestTheFirstPassAfterAStartClearsWhatAPreviousProcessRaised(t *testing.T) {
// 1945/AC24: given a module held at F under rule 2 on a node that runs it with no want, when the assigner passes, then it raises no
// neither condition for it, a neither condition this process raised for it before is cleared, and so is one a previous process
// raised (the first pass after a start), each once.
func testAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
func TestAHeldModuleRaisesNoNeitherAndClearsAStandingOne(t *testing.T) {
setup := func() (*passer, *[]asks.Condition) {
p := newPasser(buckets{delivery.AssignerWantsBucket: &memKV{values: map[string][]byte{}}, delivery.AssignerAsksBucket: &memKV{values: map[string][]byte{}}}, time.Now())
said := &[]asks.Condition{}
@@ -777,8 +665,6 @@ func mustKey(key string, err error) string {
// (ADR 0545). That record ask is not about the trunk's fingerprint: it survives every pass, so the operator is asked, and on a
// yes the want names the revised digest.
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
revised, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
@@ -827,354 +713,3 @@ func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing
t.Error("the answered record ask was taken away")
}
}
// 1126 (ADR 0549 §5): until the third milestone a pair the controller's assignments and the catalogue's placement do not
// both name gets no want and one condition, raised once and cleared once both name it; before any placement was read the
// assignments are the one source and nothing is raised.
func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 26, 18, 0, 0, 0, time.UTC))
var said []asks.Condition
p.say = func(c asks.Condition) error { said = append(said, c); return nil }
assigned := func(node, module string) {
raw, _ := json.Marshal(map[string]string{"node": node, "module": module})
p.w.put(provider.AssignmentsBucket, node+"."+module, raw)
}
place := func(node string, modules ...string) {
pl := delivery.Placement{Node: node}
for _, m := range modules {
pl.Modules = append(pl.Modules, delivery.PlacedModule{Module: m})
}
// One node key and a key per module (research 072 row 37); a module taken off leaves its key, which the node
// key no longer names.
values, err := delivery.PlacementValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
p.w.put(delivery.PlacementBucket, key, raw)
}
}
assigned("g14", "a")
assigned("g14", "b")
if got, _ := p.w.assigned(); !slices.Equal(got["g14"], []string{"a", "b"}) || len(p.sayPlacement(p.w.placementDiffers())) != 0 || len(said) != 0 {
t.Fatalf("before any placement: assigned %v, said %v", got, said)
}
none := func(context.Context, string) ([]byte, error) { return nil, errNotThere }
if m := p.w.mesh(context.Background(), none); m.Input != nil || m.InputUnread != "" {
t.Errorf("before any placement the check composes over %+v (%q)", m.Input, m.InputUnread)
}
place("g14", "a", "c")
got, _ := p.w.assigned()
frozen := p.sayPlacement(p.w.placementDiffers())
p.sayPlacement(p.w.placementDiffers())
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
!strings.Contains(said[1].Explanation, "the seat mesh-placement's placement names c on g14") {
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
}
place("g14", "a", "b", "c")
assigned("g14", "c")
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 || said[2].State != asks.StateCleared || said[3].State != asks.StateCleared {
t.Errorf("both agree again: frozen %v, said %+v", frozen, said)
}
// 1126 (ADR 0547 §2): placed, the check composes over the input the nodes compose from (frombus), once each placed node
// wrote its record; until then it says why and composes the assignments alone.
if m := p.w.mesh(context.Background(), none); m.Input != nil || !strings.Contains(m.InputUnread, "g14") {
t.Errorf("a placed node with no record: input %+v, unread %q", m.Input, m.InputUnread)
}
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14", TunnelKey: "k"}})
p.w.put(delivery.HaveBucket, "g14.node", rec)
m := p.w.mesh(context.Background(), none)
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
}
// Issue 774 (ADR 0549 §5, §6): once the controller's lease is handed away the placement alone names where modules
// run: a pair in the placement only gets its want and no condition, and one in the assignments only is no module of
// the node. A lease that goes back to a controller brings the comparison back.
place("n2", "d")
assigned("g14", "e")
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`))
got, all := p.w.assigned()
if !slices.Equal(got["n2"], []string{"d"}) || !slices.Equal(got["g14"], []string{"a", "b", "c"}) || len(all) != 0 {
t.Errorf("handed away: assigned %v, assignments %v; want the placement alone", got, all)
}
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 {
t.Errorf("handed away: frozen %v, said %+v; want no condition", frozen, said[4:])
}
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"instance":"novox/1","epoch":58}`))
if frozen := p.sayPlacement(p.w.placementDiffers()); !frozen["n2.d"] || !frozen["g14.e"] {
t.Errorf("a controller's lease again: frozen %v; want both pairs held", frozen)
}
}
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
// names no composition; a have waiting for a yes raises the record ask once, and the yes is written into the want and
// kept there while the want stays at its build.
func aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t *testing.T) {
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV, delivery.AssignerWantsBucket: wantsKV}, time.Now())
p.logf = t.Logf
put := func(bucket, key string, v []byte) { p.w.put(bucket, key, v) }
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
put(delivery.ModulesBucket, "web", mod) // the trunk's build is not done: rule 1 holds g14 at fpOld
put(provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-1"}`))
pl, _ := delivery.EncodePlacement(delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
put(delivery.PlacementBucket, "placement.g14", pl)
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
put(delivery.PresenceBucket, "g14", on)
have, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", have)
wkey, _ := delivery.WantKey("g14", "web")
want, _ := delivery.EncodeWant(delivery.Want{FP: fpOld, Composed: was})
wantsKV.values[wkey] = want
put(delivery.AssignerWantsBucket, wkey, want)
sync := func() {
for k, v := range wantsKV.values {
put(delivery.AssignerWantsBucket, k, v)
}
}
p.wants(context.Background())
p.nodeWants(context.Background())
sync()
got, err := delivery.DecodeWant(wantsKV.values[wkey])
if err != nil || got.FP != fpOld || got.Composed != "" || !got.Self {
t.Fatalf("the placed node's want is %+v (%v), want it at %s, the node composing it", got, err, fpOld)
}
nkey, _ := delivery.WantNodeKey("g14")
if nw, err := delivery.DecodeNodeWant(wantsKV.values[nkey]); err != nil || nw.Composed != "" || !nw.Self {
t.Fatalf("the placed node's own want is %+v (%v), want one naming no composition", nw, err)
}
waits, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server"})
put(delivery.HaveBucket, "g14.web", waits)
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
p.wants(context.Background())
if _, there := asksKV.values[akey]; !there {
t.Fatal("a have waiting for a yes raised no record ask")
}
// Issue 774, 2026-10-31: while the controller still composes the placed node's module (its lease not handed away), its
// digests are not the one the node's have waits on; the ask stays, and a pass that changes nothing puts nothing: on novox
// the asks of postgres, keycloak and mongodb were taken away and made again every 8 s, and a want put unchanged each
// pass made the node compose again each time.
for _, fp := range []string{fpTest, fpOld} {
ck, _ := delivery.ComposedKey("g14", "web", fp)
cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedUnchanged})
put(delivery.ControllerComposedBucket, ck, cv)
}
asked := asksKV.values[akey]
p.wants(context.Background())
if !bytes.Equal(asksKV.values[akey], asked) {
t.Fatalf("the record ask the node's have waits on was taken away or made again: %q", asksKV.values[akey])
}
wantPuts, askPuts := wantsKV.puts, asksKV.puts
p.wants(context.Background())
if wantsKV.puts != wantPuts || asksKV.puts != askPuts {
t.Fatalf("a pass that changed nothing put %d wants and %d asks", wantsKV.puts-wantPuts, asksKV.puts-askPuts)
}
// Whichever rule asks it, a want put with the value the bucket holds reaches no bucket: each put is a change its node hears.
p.put(context.Background(), delivery.AssignerWantsBucket, wkey, func() ([]byte, error) { return wantsKV.values[wkey], nil })
if wantsKV.puts != wantPuts {
t.Fatalf("a want put unchanged reached the bucket")
}
a, err := delivery.DecodeAssignerAsk(akey, asksKV.values[akey])
if err != nil {
t.Fatal(err)
}
put(delivery.AssignerAsksBucket, akey, asksKV.values[akey])
yes, _ := delivery.EncodeAnswer(delivery.Answer{Option: assign.OptionYes, Warrant: a.Ask.ID + "@2026-10-26T19:00:00Z", Words: "the operator"})
put(delivery.AnswersBucket, akey, yes)
p.wants(context.Background())
sync()
if got, err = delivery.DecodeWant(wantsKV.values[wkey]); err != nil || got.Yes != waiting || got.FP != fpOld {
t.Fatalf("after the yes the want is %+v (%v), want it to name the yes %s", got, err, waiting)
}
// The node applied it: its have runs the composition; the yes stays while the want stays at its build.
ran, _ := delivery.EncodeHave(delivery.Have{FP: fpOld, Composed: waiting, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
put(delivery.HaveBucket, "g14.web", ran)
p.wants(context.Background())
if got, _ = delivery.DecodeWant(wantsKV.values[wkey]); got.Yes != waiting {
t.Errorf("a pass after the move dropped the yes: %+v", got)
}
}
// refusingBus is the assigner's process account against a controller that refuses the calls whose verb is in refuse.
type refusingBus struct {
fakeBus
refuse map[string]string
// pins are the controller's pins its placement-export answers, by node.
mu sync.Mutex
pins map[string]map[string]delivery.Provider
}
func (b *refusingBus) setPins(node string, pins map[string]delivery.Provider) {
b.mu.Lock()
defer b.mu.Unlock()
if b.pins == nil {
b.pins = map[string]map[string]delivery.Provider{}
}
b.pins[node] = pins
}
func (b *refusingBus) Call(key string, body any) (json.RawMessage, error) {
if key == exportCall {
b.mu.Lock()
defer b.mu.Unlock()
node, _ := body.(map[string]any)["node"].(string)
answer, _ := json.Marshal(map[string]delivery.Placement{node: {V: delivery.Version, Node: node, Pins: b.pins[node]}})
return json.Marshal(map[string]any{"ok": true, "answer": json.RawMessage(answer)})
}
for verb, words := range b.refuse {
if strings.HasSuffix(key, "."+verb) {
return nil, errors.New(words)
}
}
return json.RawMessage(`{"ok":true}`), nil
}
// Issue 774: a call of the controller it refuses is a condition, never only a journal line: raised once, urgent, naming the verb,
// the node, the module and the refusal; cleared when a later call for the pair is answered, or, for a module's place, once the
// pair differed and the two sources agree.
func TestARefusedControllerCallIsACondition(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 30, 21, 4, 0, 0, time.UTC))
var conds []asks.Condition
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
bus := &refusingBus{refuse: map[string]string{"assign": `assign takes no argument "why" — it takes "build", "module", "node"; nothing was done`}}
woke := make(chan struct{}, 4)
p.seat = newSeatActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, func() bool { return false })
assignWeb := assign.ControllerCall{Verb: "assign", Node: "anchor", Module: "web", Args: map[string]any{"node": "anchor", "module": "web"}}
call := func(c assign.ControllerCall) {
p.seat.callAll([]assign.ControllerCall{c})
<-woke
}
differ := []assign.Pair{{Node: "anchor", Module: "web", Only: assign.OnlyPlacement}}
call(assignWeb)
p.sayCalls(differ)
p.sayCalls(differ)
if len(conds) != 1 || conds[0].State != asks.StateRaised || conds[0].Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
!strings.Contains(conds[0].Explanation, `takes no argument "why"`) || !strings.Contains(conds[0].Headline, "assign web on anchor") {
t.Fatalf("after a refused assign, said = %+v", conds)
}
bus.refuse = nil
call(assignWeb)
p.sayCalls(nil)
if len(conds) != 2 || conds[1].State != asks.StateCleared || conds[1].Key != conds[0].Key {
t.Fatalf("after the assign is answered, said = %+v", conds)
}
// Refused again, then assigned by hand: cleared once the pair, seen differing, agrees; not before it was seen differing.
bus.refuse = map[string]string{"assign": "refused"}
call(assignWeb)
p.sayCalls(nil)
if len(conds) != 3 || conds[2].State != asks.StateRaised {
t.Fatalf("a second refusal is not raised: %+v", conds)
}
p.sayCalls(nil)
if len(conds) != 3 {
t.Fatalf("cleared before the pair was seen differing: %+v", conds)
}
p.sayCalls(differ)
p.sayCalls(nil)
if len(conds) != 4 || conds[3].State != asks.StateCleared || !strings.Contains(conds[3].Resolved, "agree") {
t.Fatalf("the hand assign does not clear it: %+v", conds)
}
p.sayCalls(nil)
if len(conds) != 4 {
t.Fatalf("said again after the clear: %+v", conds)
}
}
// Issue 774: a refused pin's condition, and one a previous process raised, clear once the pair agrees after it was seen differing:
// the controller's pins (its placement-export, read again when its compositions change) against the placement's, the controller's
// assignments against the placement for a module's place. A start whose findings cannot be read takes them over at a later pass.
func TestARefusedCallClearsWhenTheSourcesAgree(t *testing.T) {
store := delivery.Provider{Node: "anchor", Module: "store"}
pinDB := assign.ControllerCall{Verb: "pin", Node: "laptop", Module: "db",
Args: map[string]any{"node": "laptop", "provision": "db", "from": "anchor", "module": "store"}}
placeWeb := assign.ControllerCall{Verb: "assign", Node: "anchor", Module: "web", Args: map[string]any{"node": "anchor", "module": "web"}}
webDiffers := []assign.Pair{{Node: "anchor", Module: "web", Only: assign.OnlyPlacement}}
for name, c := range map[string]struct {
call *assign.ControllerCall // refused in this process; nil: raised by a previous one
findings []string // the findings a previous process left
keysFail int // how many first reads of the findings fail
differ []assign.Pair // the pairs the two sources name apart, while they differ
agree func(*refusingBus) // the person's hand call
key string
}{
"a pin refused, pinned by hand": {call: &pinDB, agree: func(b *refusingBus) { b.setPins("laptop", map[string]delivery.Provider{"db": store}) },
key: "mesh-delivery-assigner.controller-pin.laptop.db"},
"a place refused, assigned by hand": {call: &placeWeb, differ: webDiffers, agree: func(*refusingBus) {},
key: "mesh-delivery-assigner.controller-call.anchor.web"},
"a pin raised before a start": {findings: []string{"module.laptop.db.controller-pin"}, keysFail: 1,
agree: func(b *refusingBus) { b.setPins("laptop", map[string]delivery.Provider{"db": store}) },
key: "mesh-delivery-assigner.controller-pin.laptop.db"},
"a place raised before a start": {findings: []string{"module.anchor.web.controller-call"}, differ: webDiffers, agree: func(*refusingBus) {},
key: "mesh-delivery-assigner.controller-call.anchor.web"},
} {
t.Run(name, func(t *testing.T) {
p := newPasser(buckets{}, time.Date(2026, 10, 30, 21, 4, 0, 0, time.UTC))
var conds []asks.Condition
p.say = func(c asks.Condition) error { conds = append(conds, c); return nil }
fails := c.keysFail
p.findings.Keys = func() ([]string, error) {
if fails > 0 {
fails--
return nil, errors.New("the bucket is not there yet")
}
return c.findings, nil
}
bus := &refusingBus{refuse: map[string]string{"pin": "refused", "assign": "refused"}}
woke := make(chan struct{}, 16)
p.seat = newSeatActs(bus, func() { woke <- struct{}{} }, func(string, ...any) {}, func() bool { return false })
// The placement pins db on laptop; the controller does not.
raw, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "laptop", Pins: map[string]delivery.Provider{"db": store}})
key, _ := delivery.LegacyPlacementKey("laptop")
p.w.put(delivery.PlacementBucket, key, raw)
// pass is a pass, then one more for each background read it started.
pass := func(differ []assign.Pair) {
p.sayCalls(differ)
for {
select {
case <-woke:
p.sayCalls(differ)
case <-time.After(20 * time.Millisecond):
return
}
}
}
if c.call != nil {
p.seat.callAll([]assign.ControllerCall{*c.call})
<-woke
}
pass(c.differ)
pass(c.differ)
for _, cond := range conds {
if cond.State == asks.StateCleared {
t.Fatalf("cleared while the sources differ: %+v", conds)
}
}
c.agree(bus)
p.w.put(delivery.ControllerComposedBucket, "laptop.x", []byte("moved")) // a pin changes a composition
pass(nil)
if len(conds) == 0 {
t.Fatal("nothing was raised")
}
last := conds[len(conds)-1]
if last.Key != c.key || last.State != asks.StateCleared || !strings.Contains(last.Resolved, "agree") {
t.Fatalf("the hand call does not clear it: %+v", conds)
}
n := len(conds)
pass(nil)
if len(conds) != n {
t.Errorf("said again after the clear: %+v", conds[n:])
}
})
}
}
@@ -13,7 +13,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"mesh-delivery-assigner/internal/assign"
)
@@ -47,16 +46,14 @@ type putActs struct {
snapped map[string]string
snapNo map[string]string
warranted map[string]bool
// rec records the put's warrant in the hand-act log (ADR 0565 §11).
rec handacts.Recorder
warrantNo map[string]string
said map[string]string
// snapshot takes the bus's snapshot on node; a variable so a test takes none.
snapshot func(ctx context.Context, node, module string) (string, error)
}
func newPutActs(bus caller, wake func(), logf func(string, ...any), leaseAway func() bool) *putActs {
a := &putActs{rec: handacts.Recorder{Ask: bus.Call, LeaseAway: leaseAway}, bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
func newPutActs(bus caller, wake func(), logf func(string, ...any)) *putActs {
a := &putActs{bus: bus, wake: wake, logf: logf, snapping: map[string]bool{}, snapped: map[string]string{}, snapNo: map[string]string{}, warranted: map[string]bool{}, warrantNo: map[string]string{},
said: map[string]string{}}
a.snapshot = a.snapshotTheBus
return a
@@ -186,10 +183,7 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
w.Composed, w.Granted = p.composedFor(node, module, put.FP)
if d := w.Composed; d != nil {
policy := d.Policy
chosen, _ := assign.DecodePolicies(p.w.read(assign.PoliciesBucket))
if c, ok := chosen[module]; ok {
policy, _, _, _ = catalogue.Upgrade(manifest, &c) // a person's choice at the assigner (ADR 0565 §7)
} else if policy == "" {
if policy == "" {
policy, _, _ = catalogue.DerivedUpgrade(manifest)
}
w.Recorded = policy == catalogue.PolicyRecord
@@ -219,7 +213,8 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
// from their new owner and nothing else changes.
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
var out *delivery.ComposedDigest
if d, ok := p.w.composed(node, module, fp); ok {
@@ -227,7 +222,7 @@ func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest,
}
granted := false
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
granted = g.NamesNode(node, module)
}
@@ -387,9 +382,8 @@ func (a *putActs) snapshotTheBus(ctx context.Context, node, module string) (stri
}
}
// warrant records once, in the hand-act log (the seat mesh-hand-acts', or the controller's while that seat has no holder and
// the controller holds its lease, ADR 0565 §11), that the bus's want was written on the operator's warrant: the router's
// record of the teller's ask says who chose and how (ADR 0274).
// warrant records once, in the controller's hand-act log, that the bus's want was written on the operator's warrant: the
// router's record of the teller's ask says who chose and how (ADR 0274).
func (a *putActs) warrant(id, ask string) {
a.mu.Lock()
if a.warranted[id] {
@@ -399,10 +393,10 @@ func (a *putActs) warrant(id, ask string) {
a.warranted[id] = true
a.mu.Unlock()
go func() {
if _, err := a.rec.RecordWarrant(delivery.TellerModule, ask); err != nil {
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
a.logf("the put's warrant %s is not recorded in the hand-act log: %v", ask, err)
a.mu.Lock()
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the hand-act log: %v", ask, err)
a.warrantNo[id] = fmt.Sprintf("the warrant %s is not recorded in the controller's hand-act log: %v", ask, err)
a.mu.Unlock()
a.wake()
}
File diff suppressed because it is too large Load Diff
@@ -1,668 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
)
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
// warrant and the seat's events on its stream — is the assigner's process's (seat.go), started by these writes.
// seatStore is what the seat's verbs need of the states: the runtime's, or a test's.
type seatStore interface {
store
del(state, key string) error
history(state, key string) ([]stdio.StateRevision, error)
}
func (runtimeStore) del(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
return stdio.State(state).History(key)
}
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
// `<module>.<state>`.
const (
modulesState = delivery.IndexerModule + ".modules"
haveState = "node-engine.have"
)
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
type seatTools struct {
st seatStore
ask func(key string, body any) (json.RawMessage, error)
now func() time.Time
}
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
return &seatTools{st: st, ask: ask, now: time.Now}
}
// keyed is one placement key as a verb answers it.
type keyed struct {
Placement delivery.Placement `json:"placement"`
Revision uint64 `json:"revision"`
}
// placements reads every node's placement and the revision of its node key (its legacy key's before it is converted),
// assembled from its keys in either form (delivery.PlacementKeys; research 072 row 37).
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
keys, err := s.st.keys(delivery.PlacementState)
if err != nil {
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
}
sort.Strings(keys)
held := delivery.NewPlacementKeys()
var bad []string
for _, key := range keys {
if _, _, _, err := delivery.ParsePlacementKey(key); err != nil {
continue
}
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
if err != nil {
return nil, nil, fmt.Errorf("placement key %s cannot be read: %w", key, err)
}
if !found {
continue
}
if _, err := held.Put(key, raw, rev, false); err != nil {
bad = append(bad, err.Error())
}
}
out, revs := held.AllRevisions()
if len(bad) > 0 && len(out) == 0 {
return nil, nil, fmt.Errorf("placement cannot be read: %s", strings.Join(bad, "; "))
}
return out, revs, nil
}
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
mods, err := s.st.keys(modulesState)
if err != nil {
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
}
held := map[string]bool{}
for _, k := range mods {
if name, err := delivery.ParseModulesKey(k); err == nil {
held[name] = true
}
}
haves, err := s.st.keys(haveState)
if err != nil {
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
}
recorded := map[string]bool{}
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
recorded[node] = true
}
}
}
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
}
// proposal reads one open proposal.
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
if err := delivery.CheckProposalID(id); err != nil {
return delivery.PlacementProposal{}, err
}
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
if err != nil {
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
}
if !found {
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
}
return delivery.DecodePlacementProposal(raw)
}
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
// takes as the event to compose and judge it.
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
p.Written = s.now().UTC()
for _, node := range p.Nodes() {
key, err := delivery.PlacementCandidateKey(p.ID, node)
if err != nil {
return err
}
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
if err != nil {
return err
}
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
return fmt.Errorf("the candidate %s is not written: %w", key, err)
}
}
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
}
return nil
}
// proposed is what propose and adopt answer.
type proposed struct {
Proposal delivery.PlacementProposal `json:"proposal"`
Follow string `json:"follow"`
}
func followProposal(id string) string {
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
"operator is asked to approve it once its verdict passes", id)
}
// changesOf reads propose's changes: a JSON array, given as one or as its text.
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
var raw []byte
switch c := v.(type) {
case string:
raw = []byte(c)
case nil:
return nil, errors.New("propose names its changes")
default:
var err error
if raw, err = json.Marshal(c); err != nil {
return nil, err
}
}
var out []delivery.PlacementProposalChange
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&out); err != nil {
return nil, fmt.Errorf("the changes are no list of changes: %w", err)
}
return out, nil
}
// doPropose is the verb propose (§2).
func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
changes, err := changesOf(args["changes"])
if err != nil {
return nil, err
}
for _, c := range changes {
if c.Op == delivery.OpSeed {
return nil, errors.New("the seed is adopt's alone: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
}
}
current, revs, err := s.placements()
if err != nil {
return nil, err
}
held, recorded, err := s.lays()
if err != nil {
return nil, err
}
now := s.now()
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()})
if len(findings) > 0 {
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
}
if err := s.write(p, laid); err != nil {
return nil, err
}
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
}
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
// placement-withdrawn carrying it whole.
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
why := arg(args, "why")
if why == "" {
return nil, errors.New("withdraw says why: it is kept in the event that ends the proposal")
}
p, err := s.proposal(arg(args, "id"))
if err != nil {
return nil, err
}
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
}
// doAgain is the verb again (§10): the proposal is marked, and the assigner judges it now and raises its ask again
// when its verdict passes.
func (s *seatTools) doAgain(args map[string]any) (any, error) {
p, err := s.proposal(arg(args, "id"))
if err != nil {
return nil, err
}
p.Again = s.now().UTC()
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
}
// exportCall is the controller's verb the seed is read from, by request and reply on its seat (§7).
const exportCall = "seat:" + catalogue.ControllerSeatName + ".placement-export"
// doAdopt is the verb adopt (§7): the controller's export proposed whole as the seed.
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
current, _, err := s.placements()
if err != nil {
return nil, err
}
if len(current) > 0 {
return nil, fmt.Errorf("placement is taken over already (%d keys): propose a change instead, or hand it back first", len(current))
}
if ids, err := s.st.keys(delivery.PlacementProposalsState); err == nil {
for _, id := range ids {
if p, err := s.proposal(id); err == nil && p.Seed {
return nil, fmt.Errorf("the seed %s is open: withdraw it first, or answer its ask", id)
}
}
}
raw, err := s.ask(exportCall, map[string]any{})
if err != nil {
return nil, fmt.Errorf("the controller's placement-export did not answer: %w", err)
}
export, err := assign.ExportOf(raw)
if err != nil {
return nil, err
}
held, recorded, err := s.lays()
if err != nil {
return nil, err
}
now := s.now()
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()})
if len(findings) > 0 {
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
}
if err := s.write(p, laid); err != nil {
return nil, err
}
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
}
// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease
// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or
// unreadable, is not handed away.
func (s *seatTools) leased() bool {
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
return away
}
// doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
// records the hand act and says placement-changed; every node returns to the controller's compositions.
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
if s.leased() {
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
}
why := arg(args, "why")
if why == "" {
return nil, errors.New("hand-back says why: it is recorded with the hand act")
}
if by == "" {
return nil, errors.New("the runtime names no caller: a hand-back is made by someone")
}
_, revs, err := s.placements()
if err != nil {
return nil, err
}
if len(revs) == 0 {
return nil, errors.New("there is no placement key to hand back")
}
now := s.now()
p := delivery.PlacementProposal{ID: delivery.NewProposalID(now), By: by, Why: why, At: now.UTC(), Written: now.UTC(),
HandBack: true, Base: revs, State: delivery.ProposalAnswering}
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the hand-back is not written: %w", err)
}
return proposed{Proposal: p, Follow: "the operator is asked to approve it; on the warrant every key is deleted and each " +
"node returns to the controller's compositions"}, nil
}
// placementAnswer is what the verb placement answers.
type placementAnswer struct {
Keys map[string]keyed `json:"keys,omitempty"`
Nodes []string `json:"nodes,omitempty"`
Count int `json:"count"`
History int `json:"history-per-key"`
Changes []historyEntry `json:"changes,omitempty"`
Older string `json:"older,omitempty"`
}
// historyEntry is one change of a key in the state's revisions.
type historyEntry struct {
Node string `json:"node"`
Revision uint64 `json:"revision"`
At time.Time `json:"at"`
Change delivery.PlacementChange `json:"change"`
Deleted bool `json:"deleted,omitempty"`
}
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
// changes in order from the revisions the state keeps.
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
if arg(args, "seats") == "true" {
return s.seats(arg(args, "node"))
}
all, revs, err := s.placements()
if err != nil {
return nil, err
}
node, module := arg(args, "node"), arg(args, "module")
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
for n, p := range all {
if node != "" && n != node {
continue
}
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
continue
}
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
out.Nodes = append(out.Nodes, n)
}
sort.Strings(out.Nodes)
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
return nil, fmt.Errorf("%s has no placement key", node)
}
if arg(args, "history") != "true" {
return out, nil
}
nodes := out.Nodes
if node != "" {
nodes = []string{node}
} else if module == "" {
nodes = nil
for n := range all {
nodes = append(nodes, n)
}
sort.Strings(nodes)
}
for _, n := range nodes {
revisions, err := s.nodeHistory(n)
if err != nil {
return nil, err
}
var before sketch
for _, r := range revisions {
e := historyEntry{Node: n, Revision: r.Revision, At: r.At}
after := sketch{}
if r.Op == "" || r.Op == "put" || r.Op == "PUT" {
after = sketchOf(r.Key, r.Value)
} else {
e.Deleted = true
}
e.Change = after.changeFrom(n, before)
e.Change.Before, e.Change.After = 0, r.Revision
before = after
if module != "" && !strings.Contains(" "+strings.Join(append(append(e.Change.Added, e.Change.Removed...), e.Change.Changed...), " ")+" ", " "+module+" ") {
continue
}
out.Changes = append(out.Changes, e)
}
}
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
return out, nil
}
// openProposal is one open proposal as the verb proposals answers it: the proposal (its changes, base revisions,
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
// every node's answer now.
type openProposal struct {
Proposal delivery.PlacementProposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
}
// doProposals is the verb proposals (§10).
func (s *seatTools) doProposals(args map[string]any) (any, error) {
ids, err := s.st.keys(delivery.PlacementProposalsState)
if err != nil {
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
}
cands, err := s.st.keys(delivery.PlacementCandidatesState)
if err != nil {
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
}
haves, err := s.st.keys(haveState)
if err != nil {
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
}
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
raw, _, found, err := s.st.get(haveState, k)
if err != nil || !found {
continue
}
h, err := delivery.DecodeHaveNode(raw)
if err != nil {
continue
}
for id, a := range h.Candidates {
if answers[id] == nil {
answers[id] = map[string]delivery.CandidateAnswer{}
}
answers[id][node] = a
}
}
sort.Strings(ids)
out := []openProposal{}
for _, id := range ids {
if want := arg(args, "id"); want != "" && want != id {
continue
}
p, err := s.proposal(id)
if err != nil {
return nil, err
}
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
for _, k := range cands {
pid, node, err := delivery.ParsePlacementCandidateKey(k)
if err != nil || pid != id {
continue
}
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
o.Candidates[node] = rev
}
}
out = append(out, o)
}
if arg(args, "id") != "" && len(out) == 0 {
return nil, fmt.Errorf("no proposal %s is open", arg(args, "id"))
}
return map[string]any{"proposals": out}, nil
}
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
// only from a tool of that name), with the seat's own schemas.
func (s *seatTools) tools() []stdio.Tool {
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
input := map[string]map[string]any{}
description := map[string]string{}
for _, v := range seat.Serves {
input[v.Name], description[v.Name] = v.Input, v.Description
}
run := map[string]func(map[string]any) (any, error){
catalogue.PlacementVerbPlacement: s.doPlacement,
catalogue.PlacementVerbProposals: s.doProposals,
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbAgain: s.doAgain,
catalogue.PlacementVerbAdopt: func(a map[string]any) (any, error) { return s.doAdopt(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbHandBack: func(a map[string]any) (any, error) { return s.doHandBack(a, stdio.CallerOf(a)) },
}
var out []stdio.Tool
for _, v := range seat.Serves {
out = append(out, stdio.Tool{Name: catalogue.PlacementSeat + "." + v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
}
return out
}
// seats is the verb placement with seats (ADR 0571 §4): who holds each seat and on which node, as the state seat-holders
// holds it; with node, the keys of seats held on that node.
func (s *seatTools) seats(node string) (any, error) {
keys, err := s.st.keys(delivery.SeatHoldersState)
if err != nil {
return nil, fmt.Errorf("seat-holders cannot be listed: %w", err)
}
sort.Strings(keys)
out := []delivery.SeatHolders{}
for _, key := range keys {
raw, _, found, err := s.st.get(delivery.SeatHoldersState, key)
if err != nil {
return nil, fmt.Errorf("seat-holders %s cannot be read: %w", key, err)
}
if !found {
continue
}
var sh delivery.SeatHolders
if json.Unmarshal(raw, &sh) != nil {
continue
}
if node != "" && sh.Node != node && !slices.ContainsFunc(sh.Holders, func(h delivery.SeatHolder) bool { return h.Node == node }) {
continue
}
out = append(out, sh)
}
return map[string]any{"seats": out, "holds": len(keys)}, nil
}
// keyedRevision is one revision of a node's placement key, the key it is of beside it.
type keyedRevision struct {
stdio.StateRevision
Key string
}
// nodeHistory is the revisions of a node's placement, in the order the bus wrote them: its legacy key's
// `placement.<node>` before it was converted, then its node key's `<node>.node`, whose every revision is one write of
// the node (research 072 row 37: the node key is a write's commit, so its history is the node's placement history).
func (s *seatTools) nodeHistory(node string) ([]keyedRevision, error) {
legacy, err := delivery.LegacyPlacementKey(node)
if err != nil {
return nil, err
}
nodeKey, err := delivery.PlacementNodeKey(node)
if err != nil {
return nil, err
}
var out []keyedRevision
for _, key := range []string{legacy, nodeKey} {
revisions, err := s.st.history(delivery.PlacementState, key)
if err != nil {
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
}
for _, r := range revisions {
out = append(out, keyedRevision{StateRevision: r, Key: key})
}
}
sort.SliceStable(out, func(i, j int) bool { return out[i].Revision < out[j].Revision })
return out, nil
}
// sketch is a revision of a node's placement as far as its history says a change: each module's digest, and its pins
// and holdings, which change every module (delivery.ChangeOf).
type sketch struct {
modules []delivery.PlacementRef
nodeWide string
}
// sketchOf reads one revision of a node key or a legacy key; a value that does not read places nothing.
func sketchOf(key string, value []byte) sketch {
_, _, kind, err := delivery.ParsePlacementKey(key)
if err != nil {
return sketch{}
}
wide := func(pins map[string]delivery.Provider, holds []delivery.Holding) string {
raw, _ := json.Marshal([]any{pins, holds})
return string(raw)
}
if kind == delivery.PlacementNodeKind {
n, err := delivery.DecodePlacementNode(value)
if err != nil {
return sketch{}
}
return sketch{modules: n.Modules, nodeWide: wide(n.Pins, n.Holds)}
}
p, err := delivery.DecodePlacement(value)
if err != nil {
return sketch{}
}
out := sketch{nodeWide: wide(p.Pins, p.Holds)}
for _, m := range p.Modules {
out.modules = append(out.modules, delivery.PlacementRef{Module: m.Module, Digest: m.Digest()})
}
return out
}
// changeFrom is what changed from before to s on node, as delivery.ChangeOf says it.
func (s sketch) changeFrom(node string, before sketch) delivery.PlacementChange {
c := delivery.PlacementChange{Node: node}
was := map[string]string{}
for _, r := range before.modules {
was[r.Module] = r.Digest
}
now := map[string]bool{}
nodeWide := before.nodeWide != s.nodeWide && len(before.modules) > 0
for _, r := range s.modules {
now[r.Module] = true
d, ok := was[r.Module]
switch {
case !ok:
c.Added = append(c.Added, r.Module)
case nodeWide || d != r.Digest:
c.Changed = append(c.Changed, r.Module)
}
}
for _, r := range before.modules {
if !now[r.Module] {
c.Removed = append(c.Removed, r.Module)
}
}
sort.Strings(c.Added)
sort.Strings(c.Removed)
sort.Strings(c.Changed)
return c
}
@@ -1,575 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"reflect"
"slices"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/provisions"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-assigner/internal/assign"
)
func (s memStore) del(state, key string) error {
delete(s[state].values, key)
return nil
}
func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
v, ok := s[state].values[key]
if !ok {
return nil, nil
}
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
}
// ADR 0567 §2, §7, §10: the seat's seven verbs over the states. propose lays a proposal and writes it with its candidates,
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt proposes the
// controller's export whole as the seed, once; hand-back records the hand act and deletes every key; placement and
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
func TestTheSeatsVerbs(t *testing.T) {
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
st := memStore{}
for _, state := range []string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState, modulesState, haveState} {
st[state] = &memKV{values: map[string][]byte{}}
}
for _, m := range []string{"web", "shell"} {
st[modulesState].values[m] = []byte("{}")
}
for _, n := range []string{"anchor", "laptop"} {
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
st[haveState].values[n+".node"] = raw
}
export, _ := json.Marshal(map[string]any{"anchor": anchor})
var asked []string
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
if key == exportCall {
out, _ := json.Marshal(map[string]any{"ok": true, "output": string(export)})
return out, nil
}
return json.RawMessage(`{"ok":true}`), nil
})
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
// adopt: the export proposed whole as the seed, once.
got, err := s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop")
if err != nil {
t.Fatal(err)
}
seed := got.(proposed).Proposal
if !seed.Seed || seed.ExportDigest != delivery.ExportDigest(map[string]delivery.Placement{"anchor": anchor}) || !slices.Equal(asked, []string{exportCall}) ||
len(st[delivery.PlacementCandidatesState].values) != 1 || st[delivery.PlacementProposalsState].values[seed.ID] == nil {
t.Fatalf("adopt = %+v, asked %v", seed, asked)
}
if _, err := s.doAdopt(map[string]any{"why": "again"}, "op"); err == nil || !strings.Contains(err.Error(), "is open") {
t.Errorf("a second adopt with a seed open: %v", err)
}
// withdraw and again mark the proposal; the holder ends or judges it.
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
t.Fatal(err)
}
if _, err := s.doAgain(map[string]any{"id": seed.ID}); err != nil {
t.Fatal(err)
}
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again.IsZero() {
t.Errorf("marked = %+v, %v", p, err)
}
if _, err := s.doWithdraw(map[string]any{"id": "p-none", "why": "x"}, "op"); err == nil {
t.Error("an unknown proposal was withdrawn")
}
delete(st[delivery.PlacementProposalsState].values, seed.ID) // the holder ended it
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
raw, _ := delivery.EncodePlacement(anchor)
st[delivery.PlacementState].values["placement.anchor"] = raw
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
!strings.Contains(err.Error(), "ghost") || !strings.Contains(err.Error(), "lease write") || len(st[delivery.PlacementProposalsState].values) != 0 {
t.Errorf("a refused proposal: %v", err)
}
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
t.Error("an unknown change was taken")
}
got, err = s.doPropose(map[string]any{"why": "move web", "changes": []any{
map[string]any{"op": "take-off", "node": "anchor", "module": "web"}, map[string]any{"op": "place", "node": "laptop", "module": "web"}}}, "operator@laptop")
if err != nil {
t.Fatal(err)
}
move := got.(proposed).Proposal
if move.By != "operator@laptop" || move.Base["anchor"] != uint64(len(raw)) || move.Base["laptop"] != 0 || len(st[delivery.PlacementCandidatesState].values) != 2 {
t.Fatalf("propose = %+v", move)
}
// placement and proposals read every state the seat keeps (§10); history names the stream for what is older.
reads := map[string]bool{}
pa, err := s.doPlacement(map[string]any{"module": "web", "history": "true"})
if a := pa.(placementAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, delivery.PlacementStream) {
t.Errorf("placement of web = %+v, %v", pa, err)
} else {
reads[delivery.PlacementState], reads["stream "+delivery.PlacementStream] = true, true
}
if _, err := s.doPlacement(map[string]any{"node": "laptop"}); err == nil {
t.Error("a node with no key was answered")
}
ps, err := s.doProposals(map[string]any{"id": move.ID})
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 2 {
t.Errorf("proposals = %+v, %v", ps, err)
} else {
reads[delivery.PlacementProposalsState], reads[delivery.PlacementCandidatesState] = true, true
}
raw0, err := readManifest()
if err != nil {
t.Fatal(err)
}
for _, state := range raw0.State {
if slices.Contains([]string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState}, state.Name) && !reads[state.Name] {
t.Errorf("no verb reads the seat's state %s", state.Name)
}
}
for _, stream := range raw0.Streams {
if !reads["stream "+stream.Name] {
t.Errorf("no verb reads the seat's stream %s", stream.Name)
}
}
// hand-back: the operator's alone, written as a hand-back proposal naming every key at its revision, for the assigner
// to ask; nothing is deleted by the verb.
got, err = s.doHandBack(map[string]any{"why": "back"}, "op")
if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 {
t.Errorf("hand-back = %+v, %v", got, err)
}
// Issue 774 (ADR 0567 §8): once the controller's lease is handed away a change its verbs cannot take is proposed, and
// there is no controller to hand placement back to.
st[buscompose.ControllerLease] = &memKV{values: map[string][]byte{buscompose.ControllerLeaseKey: []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)}}
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
t.Errorf("a hold after the lease write: %v", err)
}
if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") {
t.Errorf("a hand-back after the lease write: %v", err)
}
if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
t.Errorf("the seat's verbs as served: %+v", tools)
}
t.Run("1953/D4-D6: the seat mesh-wants' upgrade and bus", wantsVerbs)
t.Run("1955/L7 (ADR 0571 §6): rule 7 reads the controller's composition until the lease write, the node's own after; an unplaced node is said", func(t *testing.T) {
w := newWorld()
fp := delivery.SourcePrefix2 + strings.Repeat("2", 64)
ckey, _ := delivery.ComposedKey("ace", "web", fp)
cv, _ := delivery.EncodeComposedDigest(delivery.ComposedDigest{Digest: "sha256:" + strings.Repeat("c", 64), Judgement: delivery.JudgedNeither,
Policy: catalogue.PolicyRoll, PolicyFrom: catalogue.FromDefault})
w.putEntry(delivery.ControllerComposedBucket, ckey, cv, 1, time.Now())
pl, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "ace", Modules: []delivery.PlacedModule{{Module: "web"}}})
pkey, _ := delivery.LegacyPlacementKey("ace")
w.putEntry(delivery.PlacementBucket, pkey, pl, 1, time.Now())
for node, digest := range map[string]string{"ace": "sha256:" + strings.Repeat("a", 64), "g14": ""} {
hn, _ := delivery.EncodeHaveNode(delivery.HaveNode{Composed: digest})
hk, _ := delivery.HaveNodeKey(node)
w.putEntry(delivery.HaveBucket, hk, hn, 1, time.Now())
}
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("c", 64) || len(w.unplacedNodes()) != 0 {
t.Fatalf("before the lease write: %+v %v, unplaced %v", d, ok, w.unplacedNodes())
}
w.putEntry(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"mesh-vault","handed-by":"operator"}`), 2, time.Now())
if d, ok := w.composed("ace", "web", fp); !ok || d.Digest != "sha256:"+strings.Repeat("a", 64) {
t.Errorf("after the lease write, ace: %+v %v", d, ok)
}
if _, ok := w.composed("g14", "web", fp); ok || !slices.Equal(w.unplacedNodes(), []string{"g14"}) {
t.Errorf("after the lease write, g14 unplaced: composed %v, unplaced %v", ok, w.unplacedNodes())
}
findings := map[string][]byte{}
p := &passer{w: w, logf: func(string, ...any) {}, findings: assign.Findings{
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Delete: func(k string) error { delete(findings, k); return nil }}}
p.unplacedFindings()
if _, ok := findings[assign.UnplacedKey("g14")]; !ok || len(findings) != 1 {
t.Fatalf("findings = %v", findings)
}
pl2, _ := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}})
pkey2, _ := delivery.LegacyPlacementKey("g14")
w.putEntry(delivery.PlacementBucket, pkey2, pl2, 1, time.Now())
p.unplacedFindings()
if len(findings) != 0 {
t.Errorf("placed again: findings %v", findings)
}
})
t.Run("1955/L7 (ADR 0571 §4): seat-holders is written from placement, only the keys a change moves, and answered by placement seats", func(t *testing.T) {
writes := 0
kv := &countingKV{memKV: memKV{values: map[string][]byte{}}, writes: &writes}
p := &passer{w: newWorld(), logf: t.Logf, kvs: buckets{delivery.SeatHoldersBucket: kv}}
place := func(node string, holds ...delivery.Holding) {
pl, err := delivery.EncodePlacement(delivery.Placement{V: delivery.Version, Node: node, Modules: []delivery.PlacedModule{{Module: "web"}, {Module: "nats"}}, Holds: holds})
if err != nil {
t.Fatal(err)
}
k, _ := delivery.LegacyPlacementKey(node)
p.w.putEntry(delivery.PlacementBucket, k, pl, 1, time.Now())
}
place("ace", delivery.Holding{Claim: "mesh-broker", Scope: "mesh", Module: "nats"}, delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
place("g14", delivery.Holding{Claim: "node-shell", Scope: "node", Module: "web"})
ctx := context.Background()
p.seatHolders(ctx)
if writes != 3 || kv.values["mesh-broker"] == nil || kv.values["node-shell.ace"] == nil || kv.values["node-shell.g14"] == nil {
t.Fatalf("%d writes: %v", writes, kv.values)
}
p.seatHolders(ctx)
if writes != 3 {
t.Errorf("an unchanged placement wrote %d keys", writes-3)
}
place("g14") // g14 holds nothing now
p.seatHolders(ctx)
if writes != 3 || kv.values["node-shell.g14"] != nil || kv.values["node-shell.ace"] == nil {
t.Errorf("after g14's change: %d writes, %v", writes, kv.values)
}
st := memStore{delivery.SeatHoldersState: &memKV{values: kv.values}}
out, err := newSeatTools(st, nil).seats("ace")
if err != nil || len(out.(map[string]any)["seats"].([]delivery.SeatHolders)) != 2 {
t.Errorf("seats on ace = %+v, %v", out, err)
}
})
t.Run("research 072 row 37: the form before is converted once, a node is written as its keys, and history reads both", func(t *testing.T) {
ctx := context.Background()
kv := &revKV{values: map[string][]byte{}, revs: map[string]uint64{}}
var logs []string
p := &passer{w: newWorld(), kvs: buckets{delivery.PlacementBucket: kv}, now: time.Now,
logf: func(f string, a ...any) { logs = append(logs, fmt.Sprintf(f, a...)) }}
put := func(key string, value []byte, rev uint64) {
p.w.putEntry(delivery.PlacementBucket, key, value, rev, time.Now())
}
was := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
raw, _ := delivery.EncodePlacement(was)
put("placement.anchor", raw, kv.write("placement.anchor", raw))
p.convertPlacement(ctx)
if strings.Join(kv.log, " ") != "create anchor.web create anchor.node" || !p.convertedAll ||
!strings.Contains(strings.Join(logs, "\n"), "anchor converted from placement.anchor") {
t.Fatalf("the conversion wrote %v, said %v", kv.log, logs)
}
p.convertPlacement(ctx)
if len(kv.log) != 2 {
t.Errorf("a converted node is converted again: %v", kv.log)
}
// A write: the module keys, the node key (its commit), the mirror, then the module key no longer placed.
after := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2222}}}}
kv.log = nil
rev, err := p.writeNode(ctx, "anchor", &after)
if err != nil || strings.Join(kv.log, " ") != "create anchor.shell update anchor.node update placement.anchor delete anchor.web" || rev != kv.revs["anchor.node"] {
t.Fatalf("the write = %v, %d, %v", kv.log, rev, err)
}
if got, _ := p.seatPlacements(); !delivery.SamePlacement(got["anchor"], after) {
t.Errorf("the write is not read back at once: %+v", got)
}
// A commit refused leaves every reader on what stood: the module key written before it carries the value it replaced.
kv.refuse, kv.log = "anchor.node", nil
moved := delivery.Placement{Node: "anchor", Modules: []delivery.PlacedModule{{Module: "shell", Ports: map[string]int{"22": 2223}}}}
if _, err := p.writeNode(ctx, "anchor", &moved); err == nil || strings.Join(kv.log, " ") != "update anchor.shell" {
t.Fatalf("a refused commit = %v, %v", kv.log, err)
}
fresh, err := delivery.PlacementsOf(kv.values)
if err != nil || !delivery.SamePlacement(fresh["anchor"], after) {
t.Errorf("a reader of the keys after a refused commit reads %+v, %v", fresh["anchor"], err)
}
kv.refuse = ""
// The verb's history reads the form before, then the node key.
st := memStore{delivery.PlacementState: &memKV{values: kv.values}}
hs := newSeatTools(st, nil)
hs.st = historyStore{memStore: st, revisions: map[string][]stdio.StateRevision{
"placement.anchor": {{Key: "placement.anchor", Value: raw, Revision: 1, Op: "put"}},
"anchor.node": {{Key: "anchor.node", Value: kv.values["anchor.node"], Revision: 5, Op: "put"}, {Key: "anchor.node", Revision: 9, Op: "del"}},
}}
pa, err := hs.doPlacement(map[string]any{"node": "anchor", "history": "true"})
a, _ := pa.(placementAnswer)
if err != nil || len(a.Changes) != 3 || !slices.Equal(a.Changes[0].Change.Added, []string{"web"}) ||
!slices.Equal(a.Changes[1].Change.Added, []string{"shell"}) || !slices.Equal(a.Changes[1].Change.Removed, []string{"web"}) ||
!a.Changes[2].Deleted || !slices.Equal(a.Changes[2].Change.Removed, []string{"shell"}) {
t.Errorf("history = %+v, %v", pa, err)
}
// No placement: the mirror first, then the node key, then the module keys.
kv.log = nil
if _, err := p.writeNode(ctx, "anchor", nil); err != nil || strings.Join(kv.log, " ") != "delete placement.anchor delete anchor.node delete anchor.shell" {
t.Errorf("the delete = %v, %v", kv.log, err)
}
})
t.Run("1954/D1: a failed provision is the assigner's finding until it is provided", func(t *testing.T) {
findings := map[string][]byte{}
p := &passer{w: newWorld(), logf: func(string, ...any) {}, kvs: buckets{provisions.ProvisionsBucket: nil}, findings: assign.Findings{
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Delete: func(k string) error { delete(findings, k); return nil }}}
failed, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Failed, Why: "the role exists already"})
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", failed, 1, time.Now())
p.kvs[provisions.ProvisionsBucket] = &memKV{}
of := p.provisionsOf()
if got := of("photos")("ace"); got["ace.photos.db"].State != provisions.Failed {
t.Fatalf("provisions = %+v", got)
}
if _, ok := findings[assign.ProvisionFailedKey("ace.photos.db")]; !ok || len(findings) != 1 {
t.Fatalf("findings = %v", findings)
}
provided, _ := provisions.Encode(provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "photos", Node: "ace"},
Provider: provisions.End{Module: "postgres", Node: "novox"}, State: provisions.Provided})
p.w.putEntry(provisions.ProvisionsBucket, "ace.photos.db", provided, 2, time.Now())
p.provisionsOf()
if len(findings) != 0 {
t.Errorf("findings after provided = %v", findings)
}
// A required provision with no key: pulsed with the bound first, said once the pulse expired, ended once keyed.
ctx := context.Background()
p.kvs[delivery.PlacementPulsesBucket] = &memKV{values: map[string][]byte{}}
gone := map[string]bool{"ace.photos.s3": true}
p.provisionsMissing(ctx, gone)
if !p.pulsing("provision-missing.ace.photos.s3") || len(findings) != 0 {
t.Fatalf("first seen: pulsing %v, findings %v", p.pulsing("provision-missing.ace.photos.s3"), findings)
}
p.provisionsMissing(ctx, gone)
if len(findings) != 0 {
t.Fatal("said before its bound")
}
p.w.put(delivery.PlacementPulsesBucket, "provision-missing.ace.photos.s3", nil) // the server expired it
p.provisionsMissing(ctx, gone)
if _, ok := findings[assign.ProvisionMissingKey("ace.photos.s3")]; !ok {
t.Fatalf("past its bound: findings %v", findings)
}
p.provisionsMissing(ctx, map[string]bool{})
if len(findings) != 0 {
t.Errorf("keyed again: findings %v", findings)
}
})
}
// wantsVerbs (novox/hq ADR 0565 §7, §8, §9): upgrade answers each module's policy and its source; a person's choice is
// written with who and why, recorded as a hand act in the seat's log and said as policy-changed, and default takes it back;
// the bus is refused; compare lists the modules the controller writes another policy for; the process writes policy-split
// for a split and deletes it once they agree; bus names the node behind the indexer's newest.
func wantsVerbs(t *testing.T) {
fpOld, fpNew := delivery.SourcePrefix2+strings.Repeat("1", 64), delivery.SourcePrefix2+strings.Repeat("2", 64)
st := memStore{}
for _, state := range []string{modulesState, manifestsState, composedState, policiesState, haveState, putsState, "findings"} {
st[state] = &memKV{values: map[string][]byte{}}
}
manifest := func(name, extra string) {
raw := `{"module":"` + name + `","version":"1"` + extra + `}`
digest := delivery.ManifestDigest([]byte(raw))
st[manifestsState].values[digest], _ = delivery.EncodeManifest(delivery.Manifest{V: delivery.Version, Module: name, JSON: raw})
st[modulesState].values[name], _ = delivery.EncodeModule(delivery.Module{V: delivery.Version, FP2: fpNew, Manifest: digest, Repository: "novox/x", Commit: "c"})
}
manifest("web", "")
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}]`)
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
}
composed := func(module, policy string) {
key, _ := delivery.ComposedKey("anchor", module, fpNew)
st[composedState].values[key], _ = json.Marshal(delivery.ComposedDigest{V: delivery.Version, Digest: "sha256:" + strings.Repeat("c", 64),
Judgement: delivery.JudgedNeither, Policy: policy, PolicyFrom: catalogue.FromPerson, PolicyWhy: "chosen at the controller"})
}
composed("web", catalogue.PolicyRecord)
composed("photos", catalogue.PolicyRecord)
var asked []string
var said []string
w := newWantsTools(st, func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
return json.RawMessage(`{"ok":true}`), nil
}, func(event, id string, body any) error { said = append(said, event+" "+id); return nil })
policyOf := func(module string) policyRow {
out, err := w.doUpgrade(map[string]any{"module": module}, "op")
if err != nil {
t.Fatal(err)
}
return out.(map[string]any)["modules"].([]policyRow)[0]
}
if p := policyOf("photos"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule {
t.Errorf("photos = %+v", p)
}
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus {
t.Errorf("nats = %+v", p)
}
// compare: web rolls at the assigner and records at the controller.
out, _ := w.doUpgrade(map[string]any{"compare": "true"}, "op")
if splits := out.(map[string]any)["splits"].([]assign.Split); len(splits) != 1 || splits[0].Module != "web" || out.(map[string]any)["findings"] == nil {
t.Errorf("splits = %+v", splits)
}
// Read by watch (issue 1953 L5f): the same answer, each watched state started once, and no get per key of the
// large states (the modules, their manifests, the composed record).
watches := map[string]int{}
byWatch := newWantsTools(st, nil, nil)
byWatch.watched = stdio.WatchedStatesOver(func(state string, handle func(stdio.StateChange) error) error {
watches[state]++
for k, v := range st[state].values {
_ = handle(stdio.StateChange{Key: k, Op: stdio.OpPut, Value: v, Current: true})
}
return nil
}, func(state, key string) ([]byte, bool, error) {
watches[state+" by key"]++
v, ok := st[state].values[key]
return v, ok, nil
})
for range 2 {
wout, werr := byWatch.doUpgrade(map[string]any{"compare": "true"}, "op")
if werr != nil {
t.Fatal(werr)
}
if ws := wout.(map[string]any)["splits"].([]assign.Split); len(ws) != 1 || ws[0].Module != "web" {
t.Errorf("splits by watch = %+v", ws)
}
}
if n := watches[manifestsState+" by key"]; n == 0 || n > len(st[manifestsState].values) {
t.Errorf("the manifests were read by key %d times, want each once", n)
}
for _, state := range []string{modulesState, composedState, health.FindingsState} {
if watches[state] != 1 {
t.Errorf("%s was watched %d times, want once", state, watches[state])
}
}
// ADR 0580 clause 3: the compare reads through mesh-sdk's watched reader and the seat keeps no mirror of its own.
for f := range reflect.TypeFor[wantsTools]().Fields() {
if f.Type.Kind() == reflect.Map || (f.Type.Kind() == reflect.Pointer && f.Type.Elem().Kind() == reflect.Map) {
t.Errorf("wantsTools keeps %s, a copy of its own beside the watched reader", f.Name)
}
}
if watches[policiesState] != 0 {
t.Errorf("the seat's own small policies state was watched: %v", watches)
}
// The process says the split as its finding, and deletes it once they agree.
findings := map[string][]byte{}
p := &passer{w: newWorld(), logf: func(string, ...any) {}, findings: assign.Findings{
Put: func(k string, v []byte) error { findings[k] = v; return nil },
Delete: func(k string) error { delete(findings, k); return nil },
Keys: func() ([]string, error) { return []string{"module.gone.policy-split"}, nil }}}
for k, v := range st[composedState].values {
p.w.putEntry(delivery.ControllerComposedBucket, k, v, 1, time.Now())
}
man := map[string]catalogue.Manifest{}
for _, name := range []string{"web", "photos"} {
raw, _, _, _ := st.get(modulesState, name)
mod, _ := delivery.DecodeModule(raw)
mraw, _, _, _ := st.get(manifestsState, mod.Manifest)
kept, _ := delivery.DecodeManifest(mraw)
man[name], _ = catalogue.Decode([]byte(kept.JSON))
}
findings["module.gone.policy-split"] = []byte("{}")
p.policySplits(man, nil)
if _, ok := findings[assign.PolicySplitKey("web")]; !ok || len(findings) != 1 {
t.Errorf("findings = %v, want web's split alone (a stale one deleted)", findings)
}
// a person's record for web: written, recorded, said; the split is gone.
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "record", "why": "the operator checks it"}, "person.jochen"); err != nil {
t.Fatal(err)
}
chosen, _ := assign.DecodePolicies(st[policiesState].values)
if c := chosen["web"]; c.Policy != catalogue.PolicyRecord || c.By != "person.jochen" || !slices.Contains(asked, "seat:mesh-hand-acts.hand-act") ||
len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") {
t.Errorf("chosen %+v, asked %v, said %v", c, asked, said)
}
p.policySplits(man, chosen)
if len(findings) != 0 {
t.Errorf("findings after the choice = %v", findings)
}
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err == nil {
t.Error("the bus was given a policy")
}
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "roll"}, "op"); err == nil {
t.Error("a policy without why was set")
}
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "default", "why": "back to the module"}, "op"); err != nil || len(st[policiesState].values) != 0 {
t.Errorf("default left %v (%v)", st[policiesState].values, err)
}
// bus: the node behind the indexer's newest.
b, err := w.doBus(nil)
if err != nil {
t.Fatal(err)
}
nodes := b.(map[string]any)["nodes"].([]busNode)
if b.(map[string]any)["module"] != "nats" || len(nodes) != 2 || !nodes[0].Behind || nodes[1].Behind || !slices.Contains(asked, "seat:mesh-hand-acts.hand-acts") {
t.Errorf("bus = %+v, asked %v", b, asked)
}
if tools := w.tools(); len(tools) != 2 || tools[0].Name != "mesh-wants.upgrade" || tools[1].Run == nil {
t.Errorf("the seat's verbs as served: %+v", tools)
}
}
// readManifest is the module's manifest, read as the controller reads it.
func readManifest() (catalogue.Manifest, error) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
return catalogue.Manifest{}, err
}
return catalogue.Decode(raw)
}
// countingKV is a memKV that counts its puts.
type countingKV struct {
memKV
writes *int
}
func (c *countingKV) Put(ctx context.Context, key string, value []byte) (uint64, error) {
*c.writes++
return c.memKV.Put(ctx, key, value)
}
// revKV is a bucket with real revisions, refusing a write at a revision that is not the key's, and a write of refuse;
// log is every write in order.
type revKV struct {
jetstream.KeyValue
rev uint64
values map[string][]byte
revs map[string]uint64
log []string
refuse string
}
func (k *revKV) write(key string, value []byte) uint64 {
k.rev++
k.values[key], k.revs[key] = value, k.rev
return k.rev
}
func (k *revKV) Create(_ context.Context, key string, value []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
if _, ok := k.values[key]; ok || key == k.refuse {
return 0, errors.New("key exists")
}
k.log = append(k.log, "create "+key)
return k.write(key, value), nil
}
func (k *revKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
if k.revs[key] != revision || key == k.refuse {
return 0, errors.New("wrong last sequence")
}
k.log = append(k.log, "update "+key)
return k.write(key, value), nil
}
func (k *revKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
if key == k.refuse {
return errors.New("wrong last sequence")
}
k.log = append(k.log, "delete "+key)
delete(k.values, key)
delete(k.revs, key)
return nil
}
// historyStore is a memStore whose history is given.
type historyStore struct {
memStore
revisions map[string][]stdio.StateRevision
}
func (s historyStore) history(_, key string) ([]stdio.StateRevision, error) {
return s.revisions[key], nil
}
@@ -18,8 +18,8 @@ import (
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
// (seat_tools.go, ADR 0567 §10).
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
// putsState is the assigner's state of puts, as the runtime names a module's own state.
const putsState = "puts"
@@ -351,14 +351,7 @@ func serveTools() error {
time.Sleep(wait)
}
}()
tools := append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...)
wants := newWantsTools(runtimeStore{}, stdio.Ask, func(e, id string, b any) error {
// The seat's event, said on the seat: a claimed seat's event is never the module's own.
return stdio.SeatPublish(wantsEventSubject(e), b, id)
})
wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3)
tools = append(tools, wants.tools()...)
return stdio.Serve("", tools)
return stdio.Serve("", s.tools())
}
// shortFP is a fingerprint's first twelve hex digits.
@@ -1,323 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"git.novox.be/novox/mesh-sdk/go/health"
"mesh-delivery-assigner/internal/assign"
)
// The seat mesh-wants' verbs (novox/hq ADR 0565 §7, §8, §9, §15; issue 1953 D4-D6), served through the tool runner beside the
// put tools and mesh-placement's verbs. `upgrade` reads and sets a person's upgrade policy, kept in the state `policies`
// (each set recorded as a hand act and said as policy-changed), and with compare lists every module whose policy differs from
// the one the controller writes beside its compositions; `bus` says what a bus upgrade would do. The assigner's process reads
// `policies` when it judges a move and writes the finding policy-split while a split stands.
// The states the verbs read, as the runtime names them.
const (
policiesState = delivery.AssignerPoliciesState
manifestsState = delivery.IndexerModule + ".manifests"
composedState = "mesh-controller.composed"
// PolicyChangedEvent is the seat's event of a person's policy set or taken back.
PolicyChangedEvent = "policy-changed"
// busStepCause is the cause the bus's step is recorded under in the hand-act log.
busStepCause = "bus-upgrade"
)
// wantsTools serves the seat's verbs over a store, asking seats through ask and saying events through emit.
type wantsTools struct {
st seatStore
ask func(key string, body any) (json.RawMessage, error)
emit func(event, id string, body any) error
now func() time.Time
// watched is the copy of the large states it reads whole, each by one watch started on its first read and kept by
// its changes after (mesh-sdk's WatchedStates, novox/hq ADR 0580 clause 3), never a get per key, which timed out on
// a large state (issue 1953 L5f). Nil reads key by key.
watched *stdio.WatchedStates
}
// wantsEventSubject is the subject of one of the seat's events.
func wantsEventSubject(event string) string {
return "mesh.seat." + catalogue.WantsSeat + ".event." + event
}
func newWantsTools(st seatStore, ask func(string, any) (json.RawMessage, error), emit func(event, id string, body any) error) *wantsTools {
return &wantsTools{st: st, ask: ask, emit: emit, now: time.Now}
}
// leased says the controller's lease is handed away.
func (s *wantsTools) leased() bool {
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
return away
}
// values reads every key of a state: from the watched copy when there is one, else key by key.
func (s *wantsTools) values(state string) (map[string][]byte, error) {
if s.watched != nil {
return s.watched.Values(state)
}
return s.direct(state)
}
// direct reads a state key by key, for a small state of the seat's own.
func (s *wantsTools) direct(state string) (map[string][]byte, error) {
keys, err := s.st.keys(state)
if err != nil {
return nil, fmt.Errorf("%s cannot be listed: %w", state, err)
}
out := map[string][]byte{}
for _, k := range keys {
raw, _, found, err := s.st.get(state, k)
if err != nil {
return nil, fmt.Errorf("%s %s cannot be read: %w", state, k, err)
}
if found {
out[k] = raw
}
}
return out, nil
}
// manifest reads a manifest by its digest: kept once read when there is a watched copy, a manifest never changing
// under its digest.
func (s *wantsTools) manifest(digest string) ([]byte, bool, error) {
if s.watched != nil {
return s.watched.Once(manifestsState, digest)
}
raw, _, found, err := s.st.get(manifestsState, digest)
return raw, found, err
}
// manifests is every module the indexer holds, with its trunk manifest and fingerprint.
func (s *wantsTools) manifests() (map[string]catalogue.Manifest, map[string]delivery.Module, error) {
mods, err := s.values(modulesState)
if err != nil {
return nil, nil, err
}
out, held := map[string]catalogue.Manifest{}, map[string]delivery.Module{}
for name, raw := range mods {
m, err := delivery.DecodeModule(raw)
if err != nil {
continue
}
held[name] = m
if m.Manifest == "" {
continue
}
mraw, found, err := s.manifest(m.Manifest)
if err != nil || !found {
continue
}
kept, err := delivery.DecodeManifest(mraw)
if err != nil {
continue
}
// Decoded, never judged (ADR 0578 §3); one that does not decode is left out of the answer, as before.
if man, err := catalogue.Decode([]byte(kept.JSON)); err == nil {
out[name] = man
}
}
return out, held, nil
}
// policyRow is one module's policy as upgrade answers it.
type policyRow struct {
Module string `json:"module"`
Policy string `json:"policy"`
From string `json:"from"`
Why string `json:"why,omitempty"`
By string `json:"by,omitempty"`
}
// doUpgrade is the verb upgrade: read every module's policy or one's; with policy and why set a person's choice, or take it
// back with default; with compare list the splits.
func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
manifests, _, err := s.manifests()
if err != nil {
return nil, err
}
raw, err := s.direct(policiesState) // the seat's own small state, read whole after its own writes
if err != nil {
return nil, err
}
chosen, unreadable := assign.DecodePolicies(raw)
if arg(args, "compare") == "true" {
composed, err := s.values(composedState)
if err != nil {
return nil, err
}
splits := assign.Splits(manifests, chosen, assign.ControllerPolicies(composed))
if splits == nil {
splits = []assign.Split{}
}
out := map[string]any{"splits": splits, "empty": len(splits) == 0}
// The assigner's findings, its said conditions and its policy splits as the condition keeper reads them (§8, §15).
if found, err := s.values(health.FindingsState); err != nil {
out["findings-unread"] = err.Error()
} else {
keys := []string{}
for k := range found {
keys = append(keys, k)
}
sort.Strings(keys)
out["findings"] = keys
}
return out, nil
}
module, policy := arg(args, "module"), arg(args, "policy")
if policy != "" {
return s.setPolicy(module, policy, arg(args, "why"), by, manifests)
}
var rows []policyRow
for name, m := range manifests {
if module != "" && name != module {
continue
}
p, from, why, who := assign.PolicyOf(m, chosen)
rows = append(rows, policyRow{Module: name, Policy: p, From: from, Why: why, By: who})
}
if module != "" && len(rows) == 0 {
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
}
sort.Slice(rows, func(i, j int) bool { return rows[i].Module < rows[j].Module })
return map[string]any{"modules": rows, "unreadable": unreadable}, nil
}
// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log
// and says policy-changed. The bus is never rolled: a choice for it is refused.
func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) {
if module == "" {
return nil, errors.New("a policy is set for one module: name it")
}
m, ok := manifests[module]
if !ok {
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
}
if catalogue.ProvidesBus(m) {
return nil, fmt.Errorf("%s is the bus, which is never rolled whoever says so (novox/hq ADR 0236): its step is the put on the operator's warrant", module)
}
if why == "" {
return nil, errors.New("a person's policy says why")
}
if by == "" || by == stdio.Unproven {
return nil, errors.New("a person's policy names who chose it, and the runtime proved no caller")
}
what := fmt.Sprintf("mesh-wants.upgrade %s %s", module, policy)
if policy == "default" {
if err := s.st.del(policiesState, module); err != nil {
return nil, fmt.Errorf("the choice for %s cannot be taken back: %w", module, err)
}
} else {
p := catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: module, Policy: policy, Why: why, By: by, At: s.now().UTC()}
if err := p.Check(); err != nil {
return nil, err
}
raw, _ := json.Marshal(p)
if err := s.st.put(policiesState, module, raw); err != nil {
return nil, fmt.Errorf("the choice for %s cannot be written: %w", module, err)
}
}
rec := handacts.Recorder{Ask: s.ask, LeaseAway: s.leased}
taken, rerr := rec.Record(handacts.Request{What: what, Why: why, Cause: "upgrade-policy", Verb: "mesh-wants.upgrade"})
recorded := taken.Log
if rerr != nil {
recorded = "not recorded: " + rerr.Error()
}
body := map[string]any{"module": module, "policy": policy, "why": why, "by": by}
said := ""
if s.emit != nil {
id := PolicyChangedEvent + "." + module + "." + strconv.FormatInt(s.now().UnixNano(), 10)
if err := s.emit(PolicyChangedEvent, id, body); err != nil {
said = "not said: " + err.Error()
}
}
return map[string]any{"set": body, "hand-act": recorded, "event": said}, nil
}
// busNode is one node's bus as bus answers it.
type busNode struct {
Node string `json:"node"`
FP string `json:"fp"`
Behind bool `json:"behind"`
}
// doBus is the verb bus: the fingerprint of the bus module each node runs against the indexer's newest, the nodes behind,
// the last bus steps of the hand-act log and the assigner's puts of the bus. Read-only.
func (s *wantsTools) doBus(map[string]any) (any, error) {
manifests, held, err := s.manifests()
if err != nil {
return nil, err
}
bus := ""
for name, m := range manifests {
if catalogue.ProvidesBus(m) {
bus = name
}
}
if bus == "" {
return nil, errors.New("the indexer holds no module that provides the bus")
}
newest := held[bus].FP2
haves, err := s.values(haveState)
if err != nil {
return nil, err
}
var nodes []busNode
for key, raw := range haves {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode || module != bus {
continue
}
h, err := delivery.DecodeHave(raw)
if err != nil || h.Phase == delivery.PhaseRemoved {
continue
}
nodes = append(nodes, busNode{Node: node, FP: h.FP, Behind: newest != "" && h.FP != newest})
}
sort.Slice(nodes, func(i, j int) bool { return nodes[i].Node < nodes[j].Node })
out := map[string]any{"module": bus, "newest": newest, "nodes": nodes}
if steps, err := s.ask("seat:"+handacts.Seat+".hand-acts", map[string]any{"cause": busStepCause, "days": 90}); err != nil {
out["steps-unread"] = err.Error()
} else {
out["steps"] = steps
}
puts, err := s.values(putsState)
if err != nil {
out["puts-unread"] = err.Error()
} else {
mine := map[string]json.RawMessage{}
for k, v := range puts {
if _, module, err := delivery.ParsePutKey(k); err == nil && module == bus {
mine[k] = v
}
}
out["puts"] = mine
}
return out, nil
}
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb>, with the seat's own schemas.
func (s *wantsTools) tools() []stdio.Tool {
seat, _ := catalogue.SeatNamed(catalogue.WantsSeat)
run := map[string]func(map[string]any) (any, error){
"upgrade": func(a map[string]any) (any, error) { return s.doUpgrade(a, stdio.CallerOf(a)) },
"bus": s.doBus,
}
var out []stdio.Tool
for _, v := range seat.Serves {
out = append(out, stdio.Tool{Name: catalogue.WantsSeat + "." + v.Name, Description: v.Description, Input: v.Input, Run: run[v.Name]})
}
return out
}
@@ -13,11 +13,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
@@ -50,19 +46,6 @@ type world struct {
healthy map[string][]string
// changes counts, by bucket and key, the values this process has seen the key take: a value written again is another.
changes map[string]uint64
// meta is, by bucket and key, the revision and the time of the value held now, where the watch said them (putEntry).
meta map[string]entryMeta
// trunkBuilt is, by module, its newest done trunk build in the builder's log (putBuilt).
trunkBuilt map[string]delivery.BuildLogEntry
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
// delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner.
placement *delivery.PlacementKeys
}
// entryMeta is a value's revision in its bucket and when it was written.
type entryMeta struct {
revision uint64
at time.Time
}
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
@@ -73,30 +56,7 @@ type parsed struct {
}
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys()}
}
// putEntry is put, keeping the value's revision and the time it was written.
func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at time.Time) {
w.put(bucket, key, value)
w.mu.Lock()
defer w.mu.Unlock()
if value == nil {
delete(w.meta, bucket+"|"+key)
return
}
w.meta[bucket+"|"+key] = entryMeta{revision: revision, at: at}
if bucket == delivery.PlacementBucket {
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
}
}
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
func (w *world) metaOf(bucket, key string) entryMeta {
w.mu.Lock()
defer w.mu.Unlock()
return w.meta[bucket+"|"+key]
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{}}
}
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
@@ -106,20 +66,12 @@ func (w *world) put(bucket, key string, value []byte) {
if w.values[bucket] == nil {
w.values[bucket] = map[string][]byte{}
}
w.changes[bucket]++ // the bucket's generation (bucketGen), a delete included
if value == nil {
delete(w.values[bucket], key)
if bucket == delivery.PlacementBucket {
_, _ = w.placement.Put(key, nil, 0, true)
}
return
}
w.values[bucket][key] = value
w.changes[bucket+"|"+key]++
if bucket == delivery.PlacementBucket {
// A key or value that does not read places nothing new on its node; the indexer's read tool says it.
_, _ = w.placement.Put(key, value, 0, false)
}
if bucket == delivery.HaveBucket {
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
@@ -197,10 +149,10 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
m, err := delivery.DecodeManifest(value)
if err != nil {
p = parsed{err: err.Error()}
} else if man, err := catalogue.Decode([]byte(m.JSON)); err != nil {
// Decoded, never judged (novox/hq ADR 0578 §3): the indexer's judge judged it, and its verdict is taken from the
// module's value where it is read (world.mesh).
p = parsed{module: m.Module, err: "it does not decode: " + err.Error()}
} else if man, err := catalogue.ParseManifestForInputs([]byte(m.JSON)); err != nil {
// Read as the indexer reads it (issue 1751): a value falling back to a predefined key is judged by the merge gate
// in the catalogue, against the predefined.json at its root that this process never holds (ADR 0348 §7).
p = parsed{module: m.Module, err: err.Error()}
} else {
p = parsed{module: m.Module, manifest: man}
}
@@ -210,70 +162,8 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
return p
}
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6).
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
func (w *world) assigned() (map[string][]string, map[string]assignment) {
// A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh:
// the lease write's own check refuses that order (ADR 0547 §11).
placed, seeded := w.placed()
byAssignment, all := w.assignedByController()
if away := w.handedAway(); away && seeded {
both, _ := assign.Placed(byAssignment, placed, seeded, away)
return both, map[string]assignment{}
}
both, _ := assign.Placed(byAssignment, placed, seeded, false)
return both, all
}
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once
// the controller's lease is handed away, since the placement is then the one source.
func (w *world) placementDiffers() []assign.Pair {
placed, seeded := w.placed()
byAssignment, _ := w.assignedByController()
_, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway())
return differ
}
// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway,
// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its
// bucket not granted or not there), a controller's, expired or unreadable, is not handed away.
func (w *world) handedAway() bool {
raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey]
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil)
return away
}
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
// that does not read places nothing on its node and is said by the indexer's own read tool.
func (w *world) placed() (map[string][]string, bool) {
out := map[string][]string{}
for node, p := range w.placements() {
out[node] = p.Assigned()
}
return out, len(w.read(delivery.PlacementBucket)) > 0
}
// placements is every node's placement, assembled from its keys in either form (delivery.PlacementKeys).
func (w *world) placements() map[string]delivery.Placement {
out, _ := w.placement.AllRevisions()
return out
}
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
// so their wants name the build alone and the seventh rule does not hold them.
func (w *world) placedNodes() map[string]bool {
placed, _ := w.placed()
out := make(map[string]bool, len(placed))
for node := range placed {
out[node] = true
}
return out
}
// assignedByController is each node's modules with an open assignment, read from the controller's assignments, in
// name order.
func (w *world) assignedByController() (map[string][]string, map[string]assignment) {
out, all := map[string][]string{}, map[string]assignment{}
for _, value := range w.read(provider.AssignmentsBucket) {
var a assignment
@@ -313,18 +203,6 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
continue
}
m.Digests[name] = mod.Manifest
// The indexer's verdict on it, carried in its value (ADR 0578 §3, §5): a refused module is held and named as one
// that cannot be read, and its verdict goes into the composer's input.
if v := mod.VerdictOf(name); v != nil {
if refused, _ := delivery.TakeManifestVerdict(v); refused {
if m.Verdicts == nil {
m.Verdicts = map[string]delivery.ManifestVerdict{}
}
m.Verdicts[name] = *v
m.Unreadable[name] = v.Why()
continue
}
}
p := w.manifest(ctx, manifests, mod.Manifest)
if p.err != "" {
m.Unreadable[name] = p.err
@@ -332,127 +210,13 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
}
m.Manifests[name] = p.manifest
}
// Once the catalogue places the nodes, the check composes over the input they compose from (ADR 0547 §2, ADR 0549
// §3): one composer's input for the check and the nodes. An input that cannot be read is said, and the nodes and their
// assignments alone are composed, as before any placement.
if _, seeded := w.placed(); seeded {
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true})
if err != nil {
m.InputUnread = err.Error()
} else {
m.Input = &in
}
}
return m
}
// worldReader is what the assigner watched as frombus reads the bus: the seat's placement, the indexer's modules, each node's
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
// composition it asks resolves, and stands in for what it does not carry).
type worldReader struct {
w *world
manifests getter
}
func (r worldReader) Placements(context.Context) (map[string]delivery.Placement, error) {
return r.w.placements(), nil
}
func (r worldReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
out := map[string]delivery.NodeRecord{}
for key, value := range r.w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
out[node] = *h.Record
}
}
return out, nil
}
func (r worldReader) Modules(context.Context) (map[string]delivery.Module, error) {
return r.w.trunk(), nil
}
func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error) {
key, err := delivery.ManifestsKey(digest)
if err != nil {
return nil, err
}
raw, err := r.manifests(ctx, key)
if err != nil {
return nil, err
}
m, err := delivery.DecodeManifest(raw)
if err != nil {
return nil, err
}
return []byte(m.JSON), nil
}
// Values is one module's config values on one node as mesh-config resolved them, each a value its source lays
// (catalogue.LaidValue): the holder composes every node a proposal touches with them (ADR 0567 §9).
func (r worldReader) Values(_ context.Context, node, module string) (map[string]any, error) {
var out map[string]any
prefix := node + "." + module + "."
for key, raw := range r.w.read(buscompose.ResolvedBucket) {
name, ok := strings.CutPrefix(key, prefix)
if !ok || strings.Contains(name, ".") {
continue
}
if v, laid := catalogue.LaidValue(raw); laid {
if out == nil {
out = map[string]any{}
}
out[name] = v
}
}
return out, nil
}
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
return nil, nil
}
func (worldReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
return nil, nil, nil
}
// Build and Archive are read only for a node composing itself (frombus.Scope without Every): the check of every node
// composes from the manifests the indexer read, and reads no build and no archive.
func (worldReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) {
return nil, nil
}
func (worldReader) Archive(_ context.Context, url, _ string) ([]byte, error) {
return nil, fmt.Errorf("the composition check reads no archive (%s)", url)
}
func (r worldReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
all, err := r.Values(ctx, node, module)
var out map[string]any
for _, k := range keys {
if v, ok := all[k]; ok {
if out == nil {
out = map[string]any{}
}
out[k] = v
}
}
return out, err
}
// change is a candidate's change for the composition check, or why the composition waits for it.
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{},
Twice: map[string][]string{}, Digests: map[string]string{}, Members: map[string]string{}}
heads := map[string]string{}
for _, p := range c.Pulls {
heads[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = p.Head.Head
}
ch.Held = assign.HeldOf(w.read(keptBucket), heads)
definedIn := map[string]map[string]bool{}
for _, p := range c.Pulls {
if p.Repository == assign.SDKRepository {
@@ -501,21 +265,6 @@ func (w *world) change(ctx context.Context, manifests getter, c delivery.Candida
return ch, ""
}
// online is, by node, whether the watchman holds the node online.
func (w *world) online() map[string]bool {
out := map[string]bool{}
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
continue
}
if p, err := delivery.DecodePresence(value); err == nil {
out[node] = p.Online
}
}
return out
}
// wantsOf is the wants the assigner wrote, by module then node.
func (w *world) wantsOf() map[string]map[string]delivery.Want {
out := map[string]map[string]delivery.Want{}
@@ -584,16 +333,6 @@ func (w *world) value(bucket, key string) []byte {
return w.values[bucket][key]
}
// grantsBucket is the bucket the grants are read from now (delivery.GrantSource): the grants part's while the controller's
// lease is handed away, as the lease bucket's key says it (buscompose.LeaseHandedAway), the controller's in every other
// case, a lease not held here included.
func (w *world) grantsBucket() string {
raw := w.value(buscompose.LeaseBucket, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, raw != nil, nil)
bucket, _ := delivery.GrantSource(away)
return bucket
}
// has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool {
w.mu.Lock()
@@ -620,17 +359,7 @@ func (w *world) built(module, fp string) bool {
// composed is the controller's composition of module for node at fp: its digest and the judgement of the move to it
// (mesh-controller_composed `<node>.<module>.<hex>`), and whether it exists and can be read.
//
// From the lease write the controller's copy is no longer read (ADR 0571 §6, a named exception of clause 10 until then): a
// placed node composes its own, and its composition is the digest its own have names; an unplaced node has none, so it gets
// no want (and the pass says so, unplacedFindings).
func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool) {
if w.handedAway() {
if !w.placedNodes()[node] {
return delivery.ComposedDigest{}, false
}
return w.ownComposed(node)
}
key, err := delivery.ComposedKey(node, module, fp)
if err != nil {
return delivery.ComposedDigest{}, false
@@ -648,9 +377,6 @@ func (w *world) composed(node, module, fp string) (delivery.ComposedDigest, bool
// nodeComposed is, by node, the digest of the node's own composition (mesh-controller_composed `<node>.node`).
func (w *world) nodeComposed() map[string]delivery.ComposedDigest {
out := map[string]delivery.ComposedDigest{}
if w.handedAway() {
return out // the controller composes no node from the lease write (ADR 0571 §6)
}
for key, value := range w.read(delivery.ControllerComposedBucket) {
node, _, _, isNode, err := delivery.ParseComposedKey(key)
if err != nil || !isNode {
@@ -675,14 +401,6 @@ func (w *world) answer(key string) (delivery.Answer, bool) {
return a, err == nil
}
// bucketGen is how many times any key of the bucket was written or deleted in this process: a read of what it shows
// elsewhere is asked again only after it moves.
func (w *world) bucketGen(bucket string) uint64 {
w.mu.Lock()
defer w.mu.Unlock()
return w.changes[bucket]
}
// changed is how many values the key has taken in this process (see world.changes).
func (w *world) changed(bucket, key string) uint64 {
w.mu.Lock()
@@ -690,72 +408,29 @@ func (w *world) changed(bucket, key string) uint64 {
return w.changes[bucket+"|"+key]
}
// logWatch follows the builder's log: handle each entry it keeps, then each new one (process.LogReader.Watch).
type logWatch func(handle func(stdio.LogEntry) error) error
// putBuilt takes one entry of the builder's log in: a done trunk build's end (no ref) naming a full commit, newer than the
// module's last, becomes the module's running commit. It answers whether the commit changed.
func (w *world) putBuilt(raw []byte) bool {
e, err := delivery.DecodeBuildLogEntry(raw)
if err != nil || e.Event != delivery.BuildEndedEvent || e.Outcome != delivery.BuildDone || e.Ref != "" || len(e.Commit) != 40 {
return false
}
w.mu.Lock()
defer w.mu.Unlock()
have, ok := w.trunkBuilt[e.Module]
if ok && !e.At.After(have.At) {
return false
}
w.trunkBuilt[e.Module] = e
return !ok || have.Commit != e.Commit
}
// runningCommits is the commit of module's newest done trunk build in the builder's log (ADR 0571 clause 1): the commit a
// reader of the manifests runs at (novox/hq ADR 0482 clause 5, issue 1623); none when the log holds no such build.
func (w *world) runningCommits(module string) []string {
w.mu.Lock()
defer w.mu.Unlock()
if e, ok := w.trunkBuilt[module]; ok {
return []string{e.Commit}
}
return nil
}
// keptBucket is mesh-config's `kept`: the approved held values of open pull requests (ADR 0565 §5).
var keptBucket = "mesh-config_" + delivery.MeshConfigKeptState
// ownComposed is a placed node's own composition, as the digest its own have names (`<node>.node` of the haves, Composed):
// what rule 7 reads from the lease write. A node whose have names none is not composed yet.
func (w *world) ownComposed(node string) (delivery.ComposedDigest, bool) {
key, err := delivery.HaveNodeKey(node)
if err != nil {
return delivery.ComposedDigest{}, false
}
raw, ok := w.read(delivery.HaveBucket)[key]
if !ok {
return delivery.ComposedDigest{}, false
}
h, err := delivery.DecodeHaveNode(raw)
if err != nil || h.Composed == "" {
return delivery.ComposedDigest{}, false
}
return delivery.ComposedDigest{V: delivery.Version, Digest: h.Composed, Judgement: delivery.JudgedNeither,
Words: "the node composes its own"}, true
}
// unplacedNodes are, from the lease write, the nodes whose engine has a have and that no placement names: they get no
// want, since nothing composes them.
func (w *world) unplacedNodes() []string {
if !w.handedAway() {
return nil
}
placed := w.placedNodes()
var out []string
for key := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err == nil && isNode && !placed[node] {
out = append(out, node)
// runningCommits is, of every open assignment of module, the commit of the build it names, read from the controller's
// build record (`mesh-controller_builds` `<module>.build.<id>`), distinct and sorted: the commits a reader of the
// manifests runs at (novox/hq ADR 0482 clause 5, issue 1623). A build whose record names no full commit is left out.
func (w *world) runningCommits(ctx context.Context, get getter, module string) []string {
_, all := w.assigned()
seen := map[string]bool{}
for _, a := range all {
if a.Module != module || a.Build == "" {
continue
}
value, err := get(ctx, delivery.BuildKey(module, a.Build))
if err != nil {
continue
}
var rec delivery.BuildRecord
if json.Unmarshal(value, &rec) != nil || len(rec.Commit) != 40 {
continue
}
seen[rec.Commit] = true
}
out := make([]string, 0, len(seen))
for c := range seen {
out = append(out, c)
}
sort.Strings(out)
return out
@@ -6,9 +6,7 @@ import (
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
@@ -50,7 +48,6 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
// another repository as defined twice.
func TestTheChangeOfACandidate(t *testing.T) {
theWorldCountsTheValuesAKeyTakes(t)
f := fakeManifests{}
web := f.add(t, "web", `{"module":"web","version":"2"}`)
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
@@ -73,11 +70,10 @@ func TestTheChangeOfACandidate(t *testing.T) {
}
// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits,
// saying for what; a manifest that does not decode is the change's, not a wait (one with a field the assigner does not
// know is decoded with it set aside, novox/hq ADR 0578 §6).
// saying for what; an unreadable manifest is the change's, not a wait.
func TestAChangeWaitsForItsManifests(t *testing.T) {
f := fakeManifests{}
bad := f.add(t, "web", `{"module":"web","version":"2","requires":"x"}`)
bad := f.add(t, "web", `{"module":"web","version":"2","scope":"x"}`)
for _, c := range []delivery.CandidatePull{
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}),
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}),
@@ -117,7 +113,7 @@ func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
}
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
@@ -128,40 +124,4 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
// Issue 774: grants are read from the grants part's bucket only while the controller's lease is handed away: a grant
// only the part holds lets a node's put through then, and with a controller's lease (or none, or an unreadable one) it
// holds it, a grant only the controller holds being the one that counts instead.
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := delivery.ControllerGrantKey("web", fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{Accounts: []string{"ace.web"}})
for _, row := range []struct {
name, lease string
inPart, inController bool
bucket string
granted bool
}{
{"handed away, the part's grant", `{"handed-to":"the operator's hand-over"}`, true, false, delivery.PartGrantsBucket, true},
{"handed away, only the controller's grant", `{"handed-to":"x"}`, false, true, delivery.PartGrantsBucket, false},
{"a controller holds it, only the part's grant", `{"instance":"novox/1","epoch":3}`, true, false, delivery.ControllerGrantsBucket, false},
{"a controller holds it, its own grant", `{"instance":"novox/1"}`, false, true, delivery.ControllerGrantsBucket, true},
{"no lease read", ``, true, false, delivery.ControllerGrantsBucket, false},
{"an unreadable lease", `[1]`, true, false, delivery.ControllerGrantsBucket, false},
} {
p := newPasser(buckets{}, time.Now())
if row.lease != "" {
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(row.lease))
}
if row.inPart {
p.w.put(delivery.PartGrantsBucket, gkey, grant)
}
if row.inController {
p.w.put(delivery.ControllerGrantsBucket, gkey, grant)
}
if got := p.w.grantsBucket(); got != row.bucket {
t.Errorf("%s: grants are read from %s, want %s", row.name, got, row.bucket)
}
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted)
}
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+10 -18
View File
@@ -1,21 +1,13 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364 h1:UUDPWfUjyerptar+wPwI891jTUAtbbuXori0j3phTE4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101022905-ba8b8f28a364/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911 h1:+82ygf2hiY5Mhwfmc1xfCe9e/KY/5ObBuE2LecM37qI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101023317-8311ecb84911/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c h1:pY2YpHSSsfDPBrOEiWTNSIPn0zdnAayRmtmmyQ3hCHY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024210118-748c564cdf7c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8 h1:jA7V7hlfJAigVUvJDpKwfgd9ylQ9bqppg3rK9XgLWoA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025195051-20a6dc4920f8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8 h1:vhYk/iv43u2WGSTrFyQGDvGBxLaRITgMAiiI9DjuA10=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025200810-d4ae4b6790b8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5 h1:gE80bR0tPME7MTMBLcdP62v+F/9soVXmdcBYNMOye44=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025203528-db26f7e8b6f5/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -81,29 +81,6 @@ func HeldCondition(module, fp, node, words string, raised, needsYou bool) asks.C
return c
}
// NodeFailedCondition is the condition the assigner raises for a module whose newest fingerprint fp failed its health check on
// node while other nodes run it healthy (ADR 0557 §1, issue 1496): the failure is the node's, so nothing is held or set back and
// the node keeps its want. It names the node, the module, fp and the have's words, keyed by node and module, and is cleared
// (raised false) once the node no longer fails alone.
func NodeFailedCondition(node, module, fp string, f NodeFailure, raised bool) asks.Condition {
key := delivery.AssignerModule + ".failed-alone." + node + "." + module
subject := asks.Scope{Scope: "module", ID: module, Machine: node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("%s on %s no longer fails its health check alone.", module, node)}
}
headline := fmt.Sprintf("%s fails on %s alone", module, node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s The health check of %s at %s failed on %s (%s), while it runs healthy on %s. The failure is %s's, "+
"so the assigner holds nothing and sets no node back, and %s keeps its want. This clears once %s runs it healthy or "+
"another version is the newest.", asks.NothingToDo, module, short(fp), node, f.Words, strings.Join(f.Others, ", "), node,
node, node)}
}
// NeitherCondition is the condition of a module assigned to node that holds no want for it (issue 1945): the node's own composition
// gives up every module with a fingerprint of the new form, so nothing applies it there and its files are gone. It is raised (urgent)
// naming the module, the node and the rule holding its first want back with the rule's words, under the assigner's own name keyed by
@@ -119,14 +96,6 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
if h.Bus {
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
asks.NeedsYou, needs, module, node, h.Words)}
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
@@ -134,17 +103,6 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
"rule holds, and clears this.", asks.NothingToDo, module, node, h.Rule, h.Words)}
}
// NeitherLeftCondition clears the neither condition of module on node when it is no longer on neither list: why is "" when the
// node has a live want for it again, else why it has none (it is held, its move waits for an ask, the node no longer runs it), so
// the clear never says a want the node does not hold (issue 1945: "nats on novox has a want again" while it had none).
func NeitherLeftCondition(node, module, why string) asks.Condition {
c := NeitherCondition(node, module, Holding{}, false)
if why != "" {
c.Resolved = fmt.Sprintf("%s on %s is no longer on neither list and has no want: %s.", module, node, why)
}
return c
}
// LiftedCondition clears the condition of a held fingerprint the assigner lifted by itself, saying why (issue 1906).
func LiftedCondition(module, fp string) asks.Condition {
c := HeldCondition(module, fp, "", "", false, false)
@@ -1,14 +1,12 @@
package assign
import (
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
@@ -75,13 +73,7 @@ func TestTheRestoreAskIsValid(t *testing.T) {
// 1906/D3: the condition for a held fingerprint names the module, the node and the check's words, under the assigner's own
// name (ADR 0437 clause 9), and is cleared under the same key.
func TestTheAssignersConditions(t *testing.T) {
t.Run("the held condition names module, node and words", heldConditionNamesModuleNodeAndWords)
t.Run("a lifted hold clears its condition saying why", liftedHoldClearsItsConditionSayingWhy)
t.Run("1952/WM6: each said condition is a finding, its clear deletes it, said straight only until the take-over", saidConditionsAreFindings)
}
func heldConditionNamesModuleNodeAndWords(t *testing.T) {
func TestTheHeldConditionNamesModuleNodeAndWords(t *testing.T) {
c := HeldCondition("mail", fNew, "ace", "connection refused", true, true)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
@@ -95,7 +87,7 @@ func heldConditionNamesModuleNodeAndWords(t *testing.T) {
}
// 1906/H2: a hold lifted by itself clears the held condition under its key, saying why.
func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
func TestALiftedHoldClearsItsConditionSayingWhy(t *testing.T) {
c := LiftedCondition("mail", fNew)
if _, err := asks.EncodeCondition(c); err != nil {
t.Fatalf("the condition is refused: %v", err)
@@ -105,48 +97,6 @@ func liftedHoldClearsItsConditionSayingWhy(t *testing.T) {
}
}
// 1952/WM6 (ADR 0564 §2, issue 1952 note 16): every condition the assigner says is a finding in its own bucket, under
// `<scope>.<id>.<kind>`, written at the raise and deleted at the clear; it is said straight to the router while the take-over
// key follows the controller, and only as a finding from the take-over.
func saidConditionsAreFindings(t *testing.T) {
at := time.Date(2026, 10, 29, 21, 0, 0, 0, time.UTC)
for _, c := range []struct {
raised, cleared asks.Condition
key string
}{
{HeldCondition("mail", fNew, "ace", "refused", true, true), LiftedCondition("mail", fNew), "module.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".held"},
{NodeFailedCondition("ace", "mail", fNew, NodeFailure{Words: "x", Others: []string{"g14"}}, true), NodeFailedCondition("ace", "mail", fNew, NodeFailure{}, false), "module.ace.mail.failed-alone"},
{NeitherCondition("ace", "mail", Holding{Rule: 3, Words: "w"}, true), NeitherCondition("ace", "mail", Holding{}, false), "module.ace.mail.neither"},
{PlacementCondition(Pair{Node: "ace", Module: "mail", Only: OnlyAssignments}, true), PlacementCondition(Pair{Node: "ace", Module: "mail"}, false), "module.ace.mail.placement"},
{SeatWaitsCondition([]string{"placement"}), SeatWaitsCondition(nil), "module.mesh-delivery-assigner.seat-states"},
{BackupCondition("ace", "mail", fNew, "disk full", true), BackupCondition("ace", "mail", fNew, "", false), "module.ace.mail." + strings.TrimPrefix(fNew, delivery.SourcePrefix2) + ".backup"},
} {
puts, deletes, said := map[string]health.Finding{}, []string{}, 0
keeper := false
f := Findings{Put: func(k string, v []byte) error {
got, err := health.DecodeFinding(v)
puts[k] = got
return err
}, Delete: func(k string) error { deletes = append(deletes, k); return nil }, ToKeeper: func() bool { return keeper }, Now: func() time.Time { return at }}
say := f.Say(func(asks.Condition) error { said++; return nil })
if err := say(c.raised); err != nil {
t.Fatalf("%s: %v", c.key, err)
}
got, ok := puts[c.key]
if !ok || got.Severity != health.Severity(c.raised.Severity) || got.Headline != c.raised.Headline || got.Observer != delivery.AssignerModule ||
got.Subject.Machine != c.raised.Subject.Machine || said != 1 {
t.Fatalf("%s: finding %+v (written %v), said %d", c.key, got, puts, said)
}
keeper = true
if err := say(c.cleared); err != nil || !slices.Equal(deletes, []string{c.key}) || said != 1 {
t.Fatalf("%s: deleted %v, said %d after the take-over (%v)", c.key, deletes, said, err)
}
}
if _, _, _, err := FindingOf(asks.Condition{Key: "another.x", State: asks.StateRaised}, at); err == nil {
t.Error("a condition not the assigner's is a finding")
}
}
// 1906/D4: the record ask and the backup ask are valid asks of the router and of mesh-sdk's layout at their keys, answerable in
// the view, and name the move and the node.
func TestTheRecordAndBackupAsksAreValid(t *testing.T) {
@@ -5,7 +5,6 @@
package assign
import (
"encoding/json"
"fmt"
"slices"
"sort"
@@ -36,16 +35,6 @@ type Mesh struct {
// Digests is every trunk module's manifest digest in the indexer's manifests bucket, readable or not: what a
// composer at another mesh-sdk reads (issue 1622).
Digests map[string]string
// Input is the rest of the mesh as the nodes compose from it, filled by mesh-sdk's frombus from the catalogue's
// placement and the nodes' records (novox/hq ADR 0547 §2, ADR 0549 §3, issue 1126), so the check and the nodes share
// one composer's input; nil before any placement is read, when the nodes and their assignments alone are composed.
Input *compose.Input
// InputUnread says why that input could not be read from the bus, when it could not: the check then composes the
// nodes and their assignments alone, and says so in its warnings.
InputUnread string
// Verdicts are the indexer's refusals of trunk manifests, by module, as their values carry them (novox/hq ADR 0578
// §3): each such module is in Unreadable too, and the composer's input carries the verdict (§5).
Verdicts map[string]delivery.ManifestVerdict
}
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
@@ -63,70 +52,12 @@ type Change struct {
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
Digests map[string]string
Members map[string]string
// Held are the approved held values of the group's pull requests at the heads judged (mesh-config's `kept`, novox/hq ADR
// 0565 §5, ADR 0502 §4): each laid over its module's values on its node in the change's composition, and named.
Held []delivery.KeptValue
}
// HeldOf are the held values kept for the pull requests the candidate names, each at the head it names: one of another
// pull request, or at another head, is never laid (ADR 0502 §4). kept is mesh-config's `kept` by key.
func HeldOf(kept map[string][]byte, heads map[string]string) []delivery.KeptValue {
var out []delivery.KeptValue
for _, raw := range kept {
var k delivery.KeptValue
if json.Unmarshal(raw, &k) != nil || k.Node == "" || k.Module == "" || k.Key == "" {
continue
}
if head, ok := heads[k.Pull]; ok && head == k.Head {
out = append(out, k)
}
}
sort.Slice(out, func(i, j int) bool {
a, b := out[i], out[j]
return a.Pull+a.Node+a.Module+a.Key < b.Pull+b.Node+b.Module+b.Key
})
return out
}
// layHeld lays each held value over its module's values on its node, as a layer of its own after the rest, and answers
// the words that name each; judged first against the head's declaration of its key (catalogue.ValueProblems), a value the
// declaration refuses, or a key it does not declare, is a failure.
func layHeld(in *compose.Input, held []delivery.KeptValue) (said, fails []string) {
for _, k := range held {
var v any
if err := json.Unmarshal(k.Value, &v); err != nil {
said = append(said, fmt.Sprintf("the held value %s of %s on %s for %s cannot be read, so it is not laid: %v", k.Key, k.Module, k.Node, k.Pull, err))
continue
}
d, declared := in.Catalogue[k.Module].Settings[k.Key]
if !declared {
fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s: the head declares no such key", k.Key, k.Module, k.Node, k.Pull))
continue
}
if problems := catalogue.ValueProblems(d, v); len(problems) > 0 {
fails = append(fails, fmt.Sprintf("the held value %s of %s on %s for %s does not hold to the head's declaration: %s", k.Key, k.Module,
k.Node, k.Pull, strings.Join(problems, "; ")))
continue
}
at := slices.IndexFunc(in.Nodes, func(n compose.Node) bool { return n.Name == k.Node })
if at < 0 {
said = append(said, fmt.Sprintf("the held value %s of %s for %s names %s, which runs no module of the mesh: not laid", k.Key, k.Module, k.Pull, k.Node))
continue
}
if in.Nodes[at].Settings == nil {
in.Nodes[at].Settings = map[string][]catalogue.Layer{}
}
in.Nodes[at].Settings[k.Module] = append(in.Nodes[at].Settings[k.Module], catalogue.Layer{From: "held for " + k.Pull, Values: map[string]any{k.Key: v}})
said = append(said, fmt.Sprintf("held value laid: %s of %s on %s, held for %s at %.12s, approved by %s (novox/hq ADR 0502 §4)", k.Key, k.Module, k.Node, k.Pull, k.Head, k.Approved))
}
return said, fails
}
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
// touches no machine. A change of placement is no change of a repository (ADR 0567 §4): the seat
// mesh-placement judges it.
// touches no machine.
func Touched(m Mesh, c Change) []string {
moved := map[string]bool{}
for name := range c.Manifests {
@@ -232,11 +163,6 @@ func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.Composit
delete(changed, name)
}
after := input(changed, m.Assigned)
laid, heldFails := layHeld(&after, c.Held)
v.Warnings = append(v.Warnings, laid...)
for _, f := range heldFails {
failAt("", f)
}
for _, node := range machines {
_, before := compose.Resolve(trunk, node)
_, now := compose.Resolve(after, node)
@@ -346,18 +272,6 @@ func GrantsAlone(m Mesh, c Change, composed []string) []string {
return out
}
// withOverlay is base with every manifest of over laid over it, a copy.
func withOverlay(base, over map[string]catalogue.Manifest) map[string]catalogue.Manifest {
out := make(map[string]catalogue.Manifest, len(base)+len(over))
for name, man := range base {
out[name] = man
}
for name, man := range over {
out[name] = man
}
return out
}
func sortedKeys[V any](m map[string]V) []string {
out := make([]string, 0, len(m))
for k := range m {
@@ -1,9 +1,7 @@
package assign
import (
"encoding/json"
"slices"
"strconv"
"strings"
"testing"
@@ -40,32 +38,6 @@ func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("the verdict cannot be written: %v", err)
}
// 1953/D7 (ADR 0565 §5, ADR 0502 §4): the group's own pull request's held value at its head is laid and named, judged
// against the head's declaration; one of another pull request or another head is not laid.
web, err := catalogue.Decode([]byte(`{"module":"web","version":"2","config":{"port":{"type":"number","default":80,"description":"the port, a number","runtime":true}}}`))
if err != nil {
t.Fatal(err)
}
kept := map[string][]byte{}
for i, k := range []delivery.KeptValue{
{Pull: "novox/x#1", Head: "h1", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`81`), Approved: "op"},
{Pull: "novox/x#2", Head: "h2", Node: "g14", Module: "web", Key: "port", Value: json.RawMessage(`82`)},
{Pull: "novox/x#1", Head: "old", Node: "g14", Module: "web", Key: "other", Value: json.RawMessage(`1`)},
} {
kept[strconv.Itoa(i)], _ = json.Marshal(k)
}
held := HeldOf(kept, map[string]string{"novox/x#1": "h1"})
if len(held) != 1 || string(held[0].Value) != "81" {
t.Fatalf("held = %+v, want #1's at its head alone", held)
}
c = Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": web}, Held: held}
if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionPass || !hasWarning(v, "held value laid: port of web on g14, held for novox/x#1") {
t.Errorf("composition = %+v", v)
}
c.Held[0].Value = json.RawMessage(`"eighty"`)
if v := Compose(meshOf(), c, "x", 0); v.State != delivery.CompositionFail || !strings.Contains(v.Words, "does not hold to the head's declaration") {
t.Errorf("a held value the declaration refuses: %+v", v)
}
}
// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module.
@@ -42,25 +42,9 @@ func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
for _, name := range c.Removed {
delete(change, name)
}
in, fromClaims := plainInput(m.Assigned), true
if m.Input != nil {
// The input the nodes compose from, without its catalogue: the cases carry the manifests (issue 1126).
in = *m.Input
in.Catalogue = nil
fromClaims = len(in.Holdings) == 0
}
// Each trunk manifest the indexer refused goes in with its verdict, so the composer holds and names it (ADR 0578 §5);
// a manifest the change brings has no verdict of the indexer yet, and the checker's judge judges it (§2).
changeVerdicts := map[string]delivery.ManifestVerdict{}
for name, v := range m.Verdicts {
if _, changed := c.Digests[name]; !changed {
changeVerdicts[name] = v
}
}
changeCase := composer.Case{Name: CaseChange, Manifests: change, Verdicts: changeVerdicts}
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk, Verdicts: m.Verdicts}, changeCase}}
req := composer.Request{V: composer.Version, Input: plainInput(m.Assigned), Depth: composer.DepthResolve,
HoldingsFromClaims: true, Machines: Touched(m, c),
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, {Name: CaseChange, Manifests: change}}}
return job(SDKRepository, c.SDKHead, SDKDir, req, "the mesh-sdk head of "+c.Group)
}
@@ -147,10 +131,10 @@ type Reader struct {
// FromComposer is the composition check of a group with a mesh-sdk member, judged from the composer built at its head
// (ADR 0482 clauses 3 to 5, issues 1622 and 1623): the same failures as Compose, read from the composer's two cases, the
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk cannot decode fails too
// (the group would deliver a reader that cannot read it). A reader that cannot read a changed manifest is a warning,
// never a failure: it decodes leniently and sets a field it does not know aside (novox/hq ADR 0578 §6, which changes
// ADR 0482 §5).
// verdict naming the mesh-sdk the composer was built with; a trunk manifest the head's mesh-sdk refuses fails too (the
// group would deliver a reader that refuses it); and each reader that refuses a changed manifest the head reads fails
// it, naming the module and its words, unless the reader's mesh-sdk has no composer to ask (before issue 1621), which is
// said and judged nothing.
func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Duration) delivery.CompositionVerdict {
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: "mesh-sdk " + c.SDKHead, Took: took.Milliseconds()}
machines := Touched(m, c)
@@ -211,10 +195,6 @@ func FromComposer(m Mesh, c Change, head Ran, readers []Reader, took time.Durati
readerFails, readerWarnings := judgeReaders(c, change, readers)
fails = append(fails, readerFails...)
v.Warnings = append(v.Warnings, readerWarnings...)
if m.InputUnread != "" {
v.Warnings = append(v.Warnings, "composed over the nodes and their assignments alone: the input the nodes compose from "+
"cannot be read from the bus ("+m.InputUnread+")")
}
return finish(v, m, c, machines, fails)
}
@@ -253,10 +233,8 @@ func judgeReaders(c Change, head composer.CaseVerdict, readers []Reader) (fails,
if !head.Manifests[name].Accepted || read.Manifests[name].Accepted {
continue
}
// ADR 0578 §6: no longer a failure; the reader sets what it does not know aside, and the part that needs it takes
// the newer package (ADR 0573 §2).
warnings = append(warnings, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; it reads the rest "+
"once it takes mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, c.SDKHead))
fails = append(fails, fmt.Sprintf("the manifest of %s uses what %s (mesh-sdk %s) cannot read: %s; the group must move %s onto "+
"mesh-sdk %s", name, at, r.Ran.Verdict.SDK, read.Manifests[name].Refused, r.Module, c.SDKHead))
}
}
return fails, warnings
@@ -8,7 +8,6 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -40,10 +39,9 @@ func digestedMesh() Mesh {
// webShares is a group of mesh-sdk and mesh-catalog whose catalogue member gives web the field only the SDK head reads:
// the assigner's own reader refuses it (Unreadable), as the indexer's manifest is read in world.change.
func webShares(members map[string]string) Change {
// As an older reader refused it; the field set aside by the lenient decoder is named the same (ADR 0578 §6).
m, _ := catalogue.Decode([]byte(raws["d-web2"]))
_, err := catalogue.ParseManifest([]byte(raws["d-web2"]))
return Change{Group: "g", Heads: "0123456789ab", SDKHead: sdkHead, Manifests: map[string]catalogue.Manifest{},
Unreadable: map[string]string{"web": m.SetAside()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
Unreadable: map[string]string{"web": err.Error()}, Digests: map[string]string{"web": "d-web2"}, Members: members}
}
// run is the composer's step for a job, as the build module answers it: the request filled from the indexer's digests
@@ -102,25 +100,6 @@ func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
if !slices.Equal(req.Machines, []string{"g14"}) || req.Depth != composer.DepthResolve || !req.HoldingsFromClaims {
t.Errorf("request = machines %v, depth %q, holdings from claims %v", req.Machines, req.Depth, req.HoldingsFromClaims)
}
// 1126 (ADR 0547 §2, ADR 0549 §3): once the catalogue places the nodes, the check composes over the input the nodes
// compose from (frombus), its catalogue left to the cases, and the seats held as placement says.
m := digestedMesh()
m.Input = &compose.Input{Catalogue: map[string]catalogue.Manifest{"web": {Module: "web"}},
Nodes: []compose.Node{{Name: "g14", Assigned: []string{"web"}, Place: compose.Place{Endpoint: "g14.example:51820"}}},
Holdings: []catalogue.Held{{Claim: "web-seat", Scope: "mesh", Node: "g14", Module: "web"}}}
placedJob, err := HeadJob(m, c)
if err != nil {
t.Fatal(err)
}
placed, err := composer.Decode(placedJob.Request)
if err != nil {
t.Fatal(err)
}
if placed.Input.Catalogue != nil || len(placed.Input.Nodes) != 1 || placed.Input.Nodes[0].Place.Endpoint != "g14.example:51820" ||
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
}
}
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
@@ -142,11 +121,10 @@ func TestAFieldTheGroupAddsPassesWithTheHeadsComposer(t *testing.T) {
}
}
// 1623/AC1, AC2 as ADR 0578 §6 changes them: the group of mesh-sdk and mesh-catalog adding and using a field, without a
// controller member, passes with a warning naming the field and mesh-controller, whose running build pins an older
// mesh-sdk that cannot read it: a reader sets a field it does not know aside, so it fails nothing. With a controller
// member pinned to the SDK head it passes with no such warning.
func TestAFieldTheRunningControllerCannotReadIsAWarning(t *testing.T) {
// 1623/AC1, AC2: the group of mesh-sdk and mesh-catalog adding and using a field, without a controller member, fails
// naming the field and mesh-controller, whose running build pins an older mesh-sdk; with a controller member pinned to
// the SDK head (its composer reads what the head reads) it passes.
func TestAFieldTheRunningControllerCannotReadFailsTheGroup(t *testing.T) {
c := webShares(nil)
job, _ := HeadJob(digestedMesh(), c)
head := run(t, job, asTheHeadReads)
@@ -158,12 +136,10 @@ func TestAFieldTheRunningControllerCannotReadIsAWarning(t *testing.T) {
if rjob.Fetch["commit"] != running || rjob.Dir != delivery.RootPath {
t.Fatalf("reader job = %+v", rjob)
}
// The running controller's mesh-sdk, older than the lenient decoder, refuses web2 for its field.
old := Ran{State: delivery.StepPassed, Verdict: &composer.Verdict{V: composer.Version, SDK: "v0.1.13", Cases: []composer.CaseVerdict{
{Name: CaseChange, Manifests: map[string]composer.ManifestRead{"web": {Refused: `json: unknown field "shares"`}}}}}}
old := run(t, rjob, nil) // the running controller's mesh-sdk reads web2 as the assigner's own does: refused
v := FromComposer(digestedMesh(), c, head, []Reader{{Module: "mesh-controller", Commit: running, Ran: old}}, 0)
if v.State != delivery.CompositionPass || !hasWarning(v, "shares") || !hasWarning(v, "mesh-controller at "+running) {
t.Fatalf("without a controller member: %+v, want a pass warning of the field and mesh-controller", v)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "shares") || !strings.Contains(v.Words, "move mesh-controller onto mesh-sdk "+sdkHead) {
t.Fatalf("without a controller member: %+v, want a fail naming the field and mesh-controller", v)
}
member := "2222222222222222222222222222222222222222"
pinned := run(t, rjob, asTheHeadReads)
@@ -200,26 +176,9 @@ func TestTheHeadsComposerFailingOrRefusingTheTrunkFails(t *testing.T) {
t.Fatalf("a head composer that did not run: %+v", v)
}
job, _ := HeadJob(digestedMesh(), c)
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "requires": "x"}`}
strict := map[string]string{"d-web2": asTheHeadReads["d-web2"], "d-mail": `{"module": "mail", "version": "1", "gone": true}`}
v = FromComposer(digestedMesh(), c, run(t, job, strict), nil, 0)
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "refuses the trunk's manifest of mail") {
t.Fatalf("a head that refuses a trunk manifest: %+v", v)
}
// ADR 0578 §5: a trunk manifest the indexer refused goes into the composer's input with its verdict, in both cases;
// the composer holds and names it, and the group is not blamed for it.
m := digestedMesh()
m.Verdicts = map[string]delivery.ManifestVerdict{"mail": {Module: "mail", State: delivery.VerdictRefused, Problems: []string{"it is wrong"}}}
m.Unreadable = map[string]string{"mail": m.Verdicts["mail"].Why()}
job, _ = HeadJob(m, c)
var req composer.Request
if err := json.Unmarshal(job.Request, &req); err != nil || !req.Cases[0].Verdicts["mail"].Refused() || !req.Cases[1].Verdicts["mail"].Refused() {
t.Fatalf("the request carries no verdict: %+v %v", req.Cases, err)
}
head := run(t, job, asTheHeadReads)
if trunk, _ := caseOf(head.Verdict, CaseTrunk); !strings.Contains(trunk.Manifests["mail"].Refused, "refused the manifest of mail: it is wrong") {
t.Fatalf("the composer did not name the refusal: %+v", trunk.Manifests["mail"])
}
if v = FromComposer(m, c, head, nil, 0); v.State != delivery.CompositionPass {
t.Fatalf("a refusal on the trunk blamed the group: %+v", v)
}
}
@@ -1,105 +0,0 @@
package assign
import (
"errors"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
)
// The assigner's said conditions become findings (novox/hq ADR 0564 §2, issue 1952 WM6, note 16): every condition it says
// under its own name (ADR 0437 §9) — a hold and its lift, a node failing alone, a module on neither list, a backup, a pair
// placed by one source, the seat waiting for its states — is also a finding in its own bucket `findings`, written at its
// raise and deleted at its clear, which the condition keeper raises and clears. Until the operator's take-over the router
// hears it straight from the assigner, as before; from the take-over the keeper's condition is the one said, so the
// assigner no longer says it straight (never both).
// FindingsBucket is the assigner's findings.
var FindingsBucket = health.FindingsBucket(delivery.AssignerModule)
// FindingOf is a said condition as a finding: its key `<scope>.<id>.<kind>`, from the condition's own key
// `mesh-delivery-assigner.<kind>.<id…>` (the subject's id when the key names none), its words unchanged. raised is false for
// a clear, whose key is to be deleted; value is nil then.
func FindingOf(c asks.Condition, at time.Time) (key string, value []byte, raised bool, err error) {
rest, ok := strings.CutPrefix(c.Key, delivery.AssignerModule+".")
if !ok || rest == "" {
return "", nil, false, fmt.Errorf("the condition %q is not the assigner's", c.Key)
}
kind, id, _ := strings.Cut(rest, ".")
if id == "" {
id = c.Subject.ID
}
scope := c.Subject.Scope
if scope == "" {
scope = health.ScopeModule
}
key = health.FindingKey(scope, id, kind)
if c.State == asks.StateCleared {
return key, nil, false, nil
}
if c.State != asks.StateRaised {
return "", nil, false, fmt.Errorf("the condition %s is neither raised nor cleared", c.Key)
}
summary := c.Summary
if summary == "" {
summary = c.Headline
}
var actions []health.Action
for _, a := range c.Actions {
actions = append(actions, health.Action{Label: a.Label, Verb: a.Verb, Machine: a.Machine, Arguments: a.Arguments, Level: string(a.Level)})
}
value, err = health.EncodeFinding(health.Finding{Key: key, Kind: kind, Subject: health.Subject{Scope: scope, ID: id, Machine: c.Subject.Machine},
Severity: health.Severity(c.Severity), Summary: summary, Headline: c.Headline, Explanation: c.Explanation, Resolved: c.Resolved,
Needs: c.Needs, Actions: actions, Evidence: []health.Evidence{{At: at.UTC(), Said: summary}}, Source: "assigner." + kind,
Observer: delivery.AssignerModule, At: at.UTC()})
return key, value, true, err
}
// Findings is where the assigner's findings go: the bucket's put and delete, and whether the keeper's conditions are the
// ones said (the take-over key names the keeper).
type Findings struct {
Put func(key string, value []byte) error
Delete func(key string) error
Keys func() ([]string, error)
ToKeeper func() bool
Now func() time.Time
}
// Say is say with each condition also written as a finding (or its finding deleted), and said straight to the router only
// while the take-over key follows the controller. A finding that cannot be written is the error, beside the say's.
func (f Findings) Say(say func(asks.Condition) error) func(asks.Condition) error {
return func(c asks.Condition) error {
var errs []error
if f.Put != nil {
now := time.Now
if f.Now != nil {
now = f.Now
}
key, value, raised, err := FindingOf(c, now())
switch {
case err != nil:
errs = append(errs, err)
case raised:
if err := f.Put(key, value); err != nil {
errs = append(errs, fmt.Errorf("the finding %s cannot be written: %w", key, err))
}
default:
if err := f.Delete(key); err != nil {
errs = append(errs, fmt.Errorf("the finding %s cannot be deleted: %w", key, err))
}
}
}
if f.ToKeeper == nil || !f.ToKeeper() {
if say != nil {
if err := say(c); err != nil {
errs = append(errs, err)
}
}
}
return errors.Join(errs...)
}
}
@@ -1,132 +0,0 @@
package assign
import (
"fmt"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
// the seat mesh-placement's placement (ADR 0549 §5, ADR 0567 §8). The assigner writes wants only for a pair both name:
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
// removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal
// the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not
// yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key
// is written the controller's assignments are the one source, as before. From the lease write (the controller's lease
// handed away) the placement is the one source.
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
// OnlyPlacement.
type Pair struct {
Node, Module, Only string
// Call is what became of the assigner's call of the controller's verb for the pair, in words, when it made one.
Call string
}
// The source that names a pair alone.
const (
OnlyAssignments = "the controller's assignments"
OnlyPlacement = "the seat mesh-placement's placement"
)
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. Once the controller's
// lease is handed away (handedAway, ADR 0549 §5, §6) it is the placement alone, in name order, and no pair differs: the
// assignments are then a source of nothing.
func Placed(assigned, placed map[string][]string, seeded, handedAway bool) (map[string][]string, []Pair) {
if handedAway && seeded {
out := make(map[string][]string, len(placed))
for node, modules := range placed {
out[node] = slices.Sorted(slices.Values(modules))
}
return out, nil
}
if !seeded {
return assigned, nil
}
set := func(m map[string][]string) map[string]bool {
out := map[string]bool{}
for node, modules := range m {
for _, module := range modules {
out[node+"\x00"+module] = true
}
}
return out
}
inAssigned, inPlaced := set(assigned), set(placed)
both := map[string][]string{}
var differ []Pair
for node, modules := range assigned {
for _, module := range modules {
if inPlaced[node+"\x00"+module] {
both[node] = append(both[node], module)
} else {
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyAssignments})
}
}
}
for node, modules := range placed {
for _, module := range modules {
if !inAssigned[node+"\x00"+module] {
differ = append(differ, Pair{Node: node, Module: module, Only: OnlyPlacement})
}
}
}
for _, modules := range both {
sort.Strings(modules)
}
sort.Slice(differ, func(i, j int) bool {
if differ[i].Node != differ[j].Node {
return differ[i].Node < differ[j].Node
}
return differ[i].Module < differ[j].Module
})
return both, differ
}
// PlacementCondition is the condition of a pair the two sources do not both name: raised while they differ, cleared once
// both name it or neither does.
func PlacementCondition(p Pair, raised bool) asks.Condition {
key := delivery.AssignerModule + ".placement." + p.Node + "." + p.Module
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("the controller's assignments and the seat's placement agree on %s on %s.", p.Module, p.Node)}
}
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
var needs string
if p.Only == OnlyAssignments {
needs = fmt.Sprintf("propose placing %s on %s (mesh-placement.propose), or unassign it at the controller", p.Module, p.Node)
} else {
needs = fmt.Sprintf("assign %s to %s at the controller, as the approved placement says, or propose taking it off", p.Module, p.Node)
}
if p.Call != "" {
needs += "; the assigner's call: " + p.Call
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. %s names %s on %s and the other source does not. Until ADR 0547's third milestone both must "+
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
}
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
// §1): raised naming each, cleared when none is left.
func SeatWaitsCondition(unbound []string) asks.Condition {
key := delivery.AssignerModule + ".seat-states"
subject := asks.Scope{Scope: "module", ID: delivery.AssignerModule}
if len(unbound) == 0 {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: "the seat mesh-placement holds all its states."}
}
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
Explanation: asks.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
}
@@ -1,208 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus never rolled.
// Until the lease write the controller still writes a policy beside each composition's digest; where it differs from the
// assigner's, the assigner says so as its finding `policy-split`, naming mesh-wants.upgrade as the remedy.
// PoliciesBucket is the assigner's state `policies`.
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
// DecodePolicies reads the policies state's values by key (the module); a value that cannot be read is left out, named.
func DecodePolicies(values map[string][]byte) (map[string]catalogue.PersonPolicy, []string) {
out := map[string]catalogue.PersonPolicy{}
var bad []string
for key, raw := range values {
var p catalogue.PersonPolicy
if err := json.Unmarshal(raw, &p); err != nil || p.V != catalogue.PersonPolicyVersion || p.Check() != nil || p.Module != key {
bad = append(bad, key)
continue
}
out[key] = p
}
sort.Strings(bad)
return out, bad
}
// PolicyOf is a module's policy and where it comes from, with the person's choice for it when there is one.
func PolicyOf(m catalogue.Manifest, chosen map[string]catalogue.PersonPolicy) (policy, from, why, by string) {
var c *catalogue.PersonPolicy
if p, ok := chosen[m.Module]; ok {
c = &p
}
return catalogue.Upgrade(m, c)
}
// ControllerPolicies is, by module, every policy the controller writes beside a composition's digest of it.
func ControllerPolicies(composed map[string][]byte) map[string][]string {
out := map[string][]string{}
for key, raw := range composed {
_, module, _, isNode, err := delivery.ParseComposedKey(key)
if err != nil || isNode {
continue
}
d, err := delivery.DecodeComposedDigest(raw)
if err != nil || d.Policy == "" {
continue
}
if !contains(out[module], d.Policy) {
out[module] = append(out[module], d.Policy)
sort.Strings(out[module])
}
}
return out
}
func contains(list []string, s string) bool {
for _, v := range list {
if v == s {
return true
}
}
return false
}
// Split is a module whose policy at the assigner differs from one the controller writes beside a composition of it.
type Split struct {
Module string `json:"module"`
Policy string `json:"policy"`
From string `json:"from"`
Controller []string `json:"controller"`
}
// Splits is every module of manifests the controller writes a policy for that differs from the assigner's, sorted.
func Splits(manifests map[string]catalogue.Manifest, chosen map[string]catalogue.PersonPolicy, controller map[string][]string) []Split {
var out []Split
for module, theirs := range controller {
m, ok := manifests[module]
if !ok {
continue
}
policy, from, _, _ := PolicyOf(m, chosen)
for _, p := range theirs {
if p != policy {
out = append(out, Split{Module: module, Policy: policy, From: from, Controller: theirs})
break
}
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module })
return out
}
// KindPolicySplit is the assigner's finding of a split policy (ADR 0565 §8).
const KindPolicySplit = "policy-split"
// PolicySplitKey is a module's policy-split finding's key.
func PolicySplitKey(module string) string {
return health.FindingKey(health.ScopeModule, module, KindPolicySplit)
}
// PolicySplitFinding is the finding of one split, in mesh-sdk's layout.
func PolicySplitFinding(s Split, at time.Time) ([]byte, error) {
summary := fmt.Sprintf("%s's upgrade policy is %s at the assigner (%s) and %s at the controller", s.Module, s.Policy, s.From,
strings.Join(s.Controller, " and "))
return health.EncodeFinding(health.Finding{Key: PolicySplitKey(s.Module), Kind: KindPolicySplit,
Subject: health.Subject{Scope: health.ScopeModule, ID: s.Module}, Severity: health.Warning, Summary: summary,
Headline: clipHeadline(s.Module + "'s upgrade policy is split"),
Needs: "set it with mesh-wants.upgrade, so the assigner and the controller agree",
Explanation: asks.NeedsYou + "set it with mesh-wants.upgrade, so the assigner and the controller agree." + " The controller still " +
"writes a policy beside each composition until it hands its lease away; a choice made at its upgrade verb after the copy " +
"is not the assigner's.",
Resolved: s.Module + "'s upgrade policy agrees again", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
Source: "assigner." + KindPolicySplit, Observer: delivery.AssignerModule, At: at.UTC()})
}
func clipHeadline(s string) string {
if len([]rune(s)) <= 60 {
return s
}
return string([]rune(s)[:60])
}
// KindProvisionFailed is the assigner's finding of a provision its module's want waits for that failed (ADR 0570 §4).
const KindProvisionFailed = "provision-failed"
// ProvisionFailedKey is a failed provision's finding's key, `provider.<node>.<consumer>.<provision>[.<local>].provision-failed`.
func ProvisionFailedKey(key string) string {
return health.FindingKey(health.ScopeProvider, key, KindProvisionFailed)
}
// ProvisionFailedFinding is the finding of a failed provision, naming its consumer, its provider and why.
func ProvisionFailedFinding(key string, p provisions.Provision, at time.Time) ([]byte, error) {
summary := fmt.Sprintf("%s for %s on %s failed at %s on %s: %s, so %s is not moved there until it is provided", p.Provision,
p.Consumer.Module, p.Consumer.Node, p.Provider.Module, p.Provider.Node, p.Why, p.Consumer.Module)
return health.EncodeFinding(health.Finding{Key: ProvisionFailedKey(key), Kind: KindProvisionFailed,
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: p.Consumer.Node, Also: also(p.Provider.Node, p.Consumer.Node)},
Severity: health.Warning, Summary: summary, Headline: clipHeadline(p.Consumer.Module + " waits for a provision that failed"),
Explanation: asks.NothingToDo + " " + summary + ". It clears once the provider provides it.",
Resolved: p.Provision + " for " + p.Consumer.Module + " is provided", Evidence: []health.Evidence{{At: at.UTC(), Said: p.Why}},
Source: "assigner." + KindProvisionFailed, Observer: delivery.AssignerModule, At: at.UTC()})
}
func also(provider, consumer string) []string {
if provider == "" || provider == consumer {
return nil
}
return []string{provider}
}
// KindProvisionMissing is the assigner's finding of a required provision that has had no key for longer than its bound.
const KindProvisionMissing = "provision-missing"
// ProvisionMissingKey is its key, `provider.<node>.<module>.<provision>.provision-missing`.
func ProvisionMissingKey(key string) string {
return health.FindingKey(health.ScopeProvider, key, KindProvisionMissing)
}
// ProvisionMissingFinding is the finding of a required provision with no key past bound (key `<node>.<module>.<provision>`).
func ProvisionMissingFinding(key string, bound time.Duration, at time.Time) ([]byte, error) {
parts := strings.SplitN(key, ".", 3)
if len(parts) != 3 {
return nil, fmt.Errorf("%q is not <node>.<module>.<provision>", key)
}
node, module, provision := parts[0], parts[1], parts[2]
summary := fmt.Sprintf("%s on %s requires %s and mesh-provisioner has composed no key for it in %s, so %s is not moved there: "+
"no provider of it may be placed", module, node, provision, bound, module)
return health.EncodeFinding(health.Finding{Key: ProvisionMissingKey(key), Kind: KindProvisionMissing,
Subject: health.Subject{Scope: health.ScopeProvider, ID: key, Machine: node},
Severity: health.Warning, Summary: summary, Headline: clipHeadline(module + " on " + node + " lacks " + provision),
Explanation: asks.NothingToDo + " " + summary + ". It clears once the provision has its key.",
Resolved: provision + " for " + module + " on " + node + " has its key", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
Source: "assigner." + KindProvisionMissing, Observer: delivery.AssignerModule, At: at.UTC()})
}
// KindUnplaced is, from the lease write, a node that runs but no placement names (ADR 0571 §6).
const KindUnplaced = "unplaced"
// UnplacedKey is its finding's key, `machine.<node>.unplaced`.
func UnplacedKey(node string) string {
return health.FindingKey(health.ScopeMachine, node, KindUnplaced)
}
// UnplacedFinding is the finding of a node no placement names since the lease write.
func UnplacedFinding(node string, at time.Time) ([]byte, error) {
summary := fmt.Sprintf("%s runs, but no placement names it: since the controller handed its lease away nothing composes it, "+
"so the assigner writes it no want", node)
return health.EncodeFinding(health.Finding{Key: UnplacedKey(node), Kind: KindUnplaced,
Subject: health.Subject{Scope: health.ScopeMachine, ID: node, Machine: node}, Severity: health.Warning, Summary: summary,
Headline: clipHeadline(node + " has no placement"), Needs: "place it with mesh-placement.propose.",
Explanation: asks.NeedsYou + "place it with mesh-placement.propose. " + summary + ".",
Resolved: node + " is placed", Evidence: []health.Evidence{{At: at.UTC(), Said: summary}},
Source: "assigner." + KindUnplaced, Observer: delivery.AssignerModule, At: at.UTC()})
}
@@ -1,339 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The seat mesh-placement (novox/hq ADR 0567), held by the assigner beside its wants: placement is its state
// `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
// digest.
type ProposeIn struct {
ID, By, Why string
At time.Time
Changes []delivery.PlacementProposalChange
Current map[string]delivery.Placement
Revisions map[string]uint64
Held func(module string) bool
Recorded func(node string) bool
Seed bool
ExportDigest string
// Leased says the lease write of ADR 0547 has happened (the controller's lease handed away, mesh-sdk
// buscompose.LeaseHandedAway; issue 774), after which the controller's assignments are read no more (§8): until then
// the holder keeps the controller in step and refuses what its verbs cannot take.
Leased bool
}
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change the
// controller's verbs cannot take before the lease write (§8), a change that does not fit its node, and what
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
Changes: in.Changes, Seed: in.Seed, ExportDigest: in.ExportDigest, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
var findings []string
if p.By == "" {
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
}
if p.Why == "" {
findings = append(findings, "a proposal says why: it is shown to the operator")
}
if len(in.Changes) == 0 {
findings = append(findings, "a proposal changes something: it names no change")
}
for i, c := range in.Changes {
if (c.Op == delivery.OpSeed) != in.Seed {
findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1))
continue
}
if !in.Leased {
if why := c.BeforeLease(); why != "" {
findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why))
}
}
}
laid, lay := delivery.LayChanges(in.Current, in.Changes)
findings = append(findings, lay...)
all := map[string]delivery.Placement{}
for node, pl := range in.Current {
all[node] = pl
}
out := map[string]*delivery.Placement{}
for node, pl := range laid {
if before, ok := in.Current[node]; ok && delivery.SamePlacement(before, pl) {
continue
}
pl := pl
all[node] = pl
out[node] = &pl
p.Base[node] = in.Revisions[node]
}
if len(findings) == 0 && len(out) == 0 {
findings = append(findings, "the changes leave every placement as it is")
}
held, recorded := in.Held, in.Recorded
if held == nil {
held = func(string) bool { return false }
}
if recorded == nil {
recorded = func(string) bool { return false }
}
for _, r := range delivery.CheckPlacements(all, held, recorded) {
findings = append(findings, r)
}
return p, out, findings
}
// Write is one node a warrant writes (§5): the node, its node key `<node>.node` (the commit of the node's keys,
// mesh-sdk delivery.PlacementKeys.Writes), the revision of the node's placement it is written at (0: it has none, so
// it is created) and its value (nil: it is deleted).
type Write struct {
Node, Key string
Base uint64
Value *delivery.Placement
}
// Writes is every key a proposal writes, by node order.
func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) ([]Write, error) {
var out []Write
for _, node := range p.Nodes() {
key, err := delivery.PlacementNodeKey(node)
if err != nil {
return nil, err
}
out = append(out, Write{Node: node, Key: key, Base: p.Base[node], Value: laid[node]})
}
return out, nil
}
// ControllerCall is one call of the controller's seat verbs the holder makes after writing an approved proposal, until
// the lease write (§8): assign, unassign, pin or unpin, with its arguments.
type ControllerCall struct {
Verb string
Args map[string]any
// Node and Module name the pair the call keeps in step (a pin names no module).
Node, Module string
}
// Key names the call, as the condition of its pair says it.
func (c ControllerCall) Key() string {
return fmt.Sprintf("mesh-controller.%s %s %s", c.Verb, c.Node, c.Module)
}
// ControllerCalls is every call that brings the controller's assignments and pins to the placement after: assign for a
// module placed on a node before not, unassign for one taken off, pin for a pin set or changed, unpin for one removed. In
// node order, unassigns before assigns, so a module moved between nodes is first taken off.
//
// Each call carries only the arguments the controller's seat contract declares for its verb (catalogue.ControllerVerbs):
// the controller refuses any other argument and does nothing (issue 774: a "why" added to every call had every approved
// placement refused since 2026-10-28). The proposal and its warrant are in the assigner's own record instead: the
// placement-changed event and the hand-act log.
func ControllerCalls(before map[string]delivery.Placement, after map[string]*delivery.Placement) []ControllerCall {
nodes := make([]string, 0, len(after))
for node := range after {
nodes = append(nodes, node)
}
sort.Strings(nodes)
var off, on, pins []ControllerCall
for _, node := range nodes {
was, now := before[node], delivery.Placement{}
if a := after[node]; a != nil {
now = *a
}
had, has := map[string]bool{}, map[string]bool{}
for _, m := range was.Modules {
had[m.Module] = true
}
for _, m := range now.Modules {
has[m.Module] = true
if !had[m.Module] {
on = append(on, ControllerCall{Verb: "assign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
}
}
for _, m := range was.Modules {
if !has[m.Module] {
off = append(off, ControllerCall{Verb: "unassign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
}
}
for _, provision := range sortedKeys(now.Pins) {
to := now.Pins[provision]
if old, ok := was.Pins[provision]; !ok || old != to {
pins = append(pins, ControllerCall{Verb: "pin", Node: node, Module: provision,
Args: map[string]any{"node": node, "provision": provision, "from": to.Node, "module": to.Module}})
}
}
for _, provision := range sortedKeys(was.Pins) {
if _, ok := now.Pins[provision]; !ok {
pins = append(pins, ControllerCall{Verb: "unpin", Node: node, Module: provision, Args: map[string]any{"node": node, "provision": provision}})
}
}
}
return append(append(off, on...), pins...)
}
// CallKind is the kind of the condition of a refused call: one for a module's place (assign, unassign), one for a pin.
func (c ControllerCall) CallKind() string {
if c.Verb == "pin" || c.Verb == "unpin" {
return KindControllerPin
}
return KindControllerCall
}
// The kinds of a refused controller call's condition (and finding).
const (
KindControllerCall = "controller-call"
KindControllerPin = "controller-pin"
)
// CallConditionKey is the key of the condition of a call refused for its pair (node and module, or node and provision).
func CallConditionKey(c ControllerCall) string {
return delivery.AssignerModule + "." + c.CallKind() + "." + c.Node + "." + c.Module
}
// CallRefusedCondition is the condition the assigner raises when the controller refused (or did not answer) a call that
// keeps its assignments in step with an approved placement (ADR 0567 §8, issue 774): the placement is written, the
// controller's assignments are not, so nothing new reaches the node (no bus account, no composition). It names the verb,
// the node, the module (or provision) and the controller's words, keyed by the pair, and is cleared (raised false) when a
// later call for the pair is answered, or both sources agree on it, with resolved saying which.
func CallRefusedCondition(c ControllerCall, refusal string, raised bool, resolved string) asks.Condition {
key := CallConditionKey(c)
subject := asks.Scope{Scope: "module", ID: c.Module, Machine: c.Node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: resolved}
}
headline := fmt.Sprintf("the controller refused %s %s on %s", c.Verb, c.Module, c.Node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
needs := fmt.Sprintf("call mesh-controller.%s for %s on %s by hand once the refusal's cause is fixed", c.Verb, c.Module, c.Node)
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. An approved placement was written, and the assigner's call %s to keep the controller's "+
"assignments in step was refused: %s. Until the controller holds it, the placement and the assignments differ, so "+
"the assigner writes no want for the pair and nothing new reaches %s. This clears when a call for it is answered "+
"or both sources agree.", asks.NeedsYou, needs, c.Key(), refusal, c.Node)}
}
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
func PlacementAct(p delivery.PlacementProposal) asks.Act {
return asks.Act{"placement": "write", "proposal": p.ID, "changes": delivery.ChangesDigest(p)}
}
// PlacementAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
// another asking, under an id of its own.
func PlacementAsking(p delivery.PlacementProposal, at time.Time) int {
return int(at.Unix())
}
// PlacementAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking int, now time.Time) (delivery.AssignerAsk, error) {
binds, err := asks.ActDigest(PlacementAct(p))
if err != nil {
return delivery.AssignerAsk{}, err
}
nodes := p.Nodes()
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
if p.Seed {
headline = "Take placement over from the controller?"
}
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
}
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: headline,
Explanation: brief,
Options: []asks.Option{
{ID: OptionYes, Label: "Approve", Does: "the placement of " + strings.Join(nodes, ", ") + " is written as proposed",
Level: asks.Approve, Binds: binds},
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(asks.ApproveLasts),
OnExpiry: "nothing is written, and the proposal ends",
About: key,
}, Asking: asking}, nil
}
// ExportOf reads the controller's answer to placement-export (§7): its verb answers what the command printed, the
// JSON of every node's placement, in `output` (the controller's verbAnswer; `answer` when it carries it as data). A
// refused or unreadable answer is an error naming it.
func ExportOf(raw []byte) (map[string]delivery.Placement, error) {
var a struct {
Output string `json:"output"`
OK *bool `json:"ok"`
Answer json.RawMessage `json:"answer"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the controller's placement-export answered no verb answer: %w", err)
}
if a.OK != nil && !*a.OK {
return nil, fmt.Errorf("the controller's placement-export refused: %s", strings.TrimSpace(a.Output))
}
var byNode map[string]json.RawMessage
if len(a.Answer) > 0 && string(a.Answer) != "null" {
if err := json.Unmarshal(a.Answer, &byNode); err != nil {
return nil, fmt.Errorf("the controller's placement-export is no placement by node: %w", err)
}
} else if err := json.NewDecoder(strings.NewReader(a.Output)).Decode(&byNode); err != nil {
// The command prints its JSON first; what its standard error said after it is left.
return nil, fmt.Errorf("the controller's placement-export printed no placement by node: %w", err)
}
out := map[string]delivery.Placement{}
for node, rawP := range byNode {
p, err := delivery.DecodePlacement(rawP)
if err != nil {
return nil, fmt.Errorf("the controller's placement-export of %s cannot be read: %w", node, err)
}
if p.Node != node {
return nil, fmt.Errorf("the controller's placement-export names %s under %s", p.Node, node)
}
out[node] = p
}
if len(out) == 0 {
return nil, fmt.Errorf("the controller's placement-export names no node")
}
return out, nil
}
// SeedChanges is the seed's changes: every node's placement of the export, whole, in node order.
func SeedChanges(export map[string]delivery.Placement) []delivery.PlacementProposalChange {
var out []delivery.PlacementProposalChange
for _, node := range sortedKeys(export) {
p := export[node]
out = append(out, delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: node, Whole: &p})
}
return out
}
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
rest := p
rest.Changes, rest.Base = nil, map[string]uint64{}
for _, c := range p.Changes {
if _, done := written[c.Node]; !done {
rest.Changes = append(rest.Changes, c)
}
}
for node, base := range p.Base {
if _, done := written[node]; !done {
rest.Base[node] = base
}
}
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", delivery.ProposalAnswering
return rest, len(rest.Changes) > 0
}
@@ -1,238 +0,0 @@
package assign
import (
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0567 §2, §5, §7, §8: what the seat's holder decides of a proposal: every refusal named at once, a valid one laid
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
func TestThePlacementSeatDecides(t *testing.T) {
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Decided: delivery.NodeDecided{Hub: true, Address: "10.0.0.1"},
Modules: []delivery.PlacedModule{{Module: "store", Ports: map[string]int{"5432": 5432}}, {Module: "web"}}}
laptop := delivery.Placement{V: delivery.Version, Node: "laptop", Decided: delivery.NodeDecided{Address: "10.0.0.2"}, Modules: []delivery.PlacedModule{{Module: "shell"}}}
in := func(changes ...delivery.PlacementProposalChange) ProposeIn {
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
Current: map[string]delivery.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
Held: func(m string) bool { return m != "ghost" },
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
}
move := []delivery.PlacementProposalChange{{Op: delivery.OpTakeOff, Node: "anchor", Module: "web"}, {Op: delivery.OpPlace, Node: "laptop", Module: "web"}}
p, laid, findings := Propose(in(move...))
if len(findings) != 0 || p.State != delivery.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
!slices.Equal(laid["laptop"].Assigned(), []string{"shell", "web"}) || !slices.Equal(laid["anchor"].Assigned(), []string{"store"}) {
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
}
if _, err := delivery.EncodePlacementProposal(p); err != nil {
t.Fatalf("the proposal cannot be written: %v", err)
}
for name, c := range map[string]struct {
in ProposeIn
want string
}{
"a port before the lease": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}}), "lease write"},
"a holding": {in(delivery.PlacementProposalChange{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "mesh-store", Scope: "mesh"}), "lease write"},
"a binding": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}}), "lease write"},
"a config place": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}}), "lease write"},
"a config access": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}}), "lease write"},
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "adopt's alone"},
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
} {
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
}
}
// A module's config values may be proposed before the lease write (§13); once the lease is written every change is
// the state's alone, and none waits.
if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 {
t.Errorf("config values before the lease write: %v", f)
}
for _, c := range []delivery.PlacementProposalChange{
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
} {
leased := in(c)
leased.Leased = true
if _, _, f := Propose(leased); len(f) != 0 {
t.Errorf("after the lease write %s is refused: %v", c.Op, f)
}
}
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
len(rest.Base) != 1 || rest.State != delivery.ProposalAnswering || rest.Must != nil {
t.Errorf("the rest = %+v, %v", rest, ok)
}
if _, ok := Rest(p, laid); ok {
t.Error("a proposal written whole leaves something to write")
}
// The seat says it waits while one of its states is not bound, naming it, and clears once all are.
for _, unbound := range [][]string{{delivery.PlacementPulsesBucket}, nil} {
c := SeatWaitsCondition(unbound)
if _, err := asks.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == asks.StateRaised) ||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+delivery.PlacementPulsesBucket)) {
t.Errorf("waiting for %v: %+v, %v", unbound, c, err)
}
}
// Issue 1126: the seed places the private network, which no repository holds; it is the mesh's own, and passes.
wired := laptop
wired.Modules = append(slices.Clone(laptop.Modules), delivery.PlacedModule{Module: "mesh-wireguard"})
seed := in(SeedChanges(map[string]delivery.Placement{"laptop": wired})...)
seed.Seed, seed.ExportDigest, seed.Current, seed.Revisions = true, "sha256:x", map[string]delivery.Placement{}, map[string]uint64{}
seed.Held = func(m string) bool { return m != "mesh-wireguard" } // the indexer holds every module but it
if _, _, f := Propose(seed); len(f) != 0 {
t.Errorf("a seed placing the private network is refused: %v", f)
}
writes, err := Writes(p, laid)
if err != nil || len(writes) != 2 || writes[0].Key != "anchor.node" || writes[0].Base != 4 || writes[1].Base != 9 {
t.Fatalf("writes = %+v, %v", writes, err)
}
pinned := *laid["laptop"]
pinned.Pins = map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
calls := ControllerCalls(map[string]delivery.Placement{"anchor": anchor, "laptop": laptop},
map[string]*delivery.Placement{"anchor": laid["anchor"], "laptop": &pinned})
var said []string
for _, c := range calls {
said = append(said, c.Key())
}
if !slices.Equal(said, []string{"mesh-controller.unassign anchor web", "mesh-controller.assign laptop web", "mesh-controller.pin laptop db"}) {
t.Errorf("the controller's calls = %v", said)
}
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
t.Fatalf("the ask = %+v, %v", a, err)
}
q := p
q.Changes = append(slices.Clone(p.Changes), delivery.PlacementProposalChange{Op: delivery.OpUnpin, Node: "laptop", Provision: "x"})
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
t.Error("the ask's yes binds changes it was not asked about")
}
export, _ := json.Marshal(map[string]any{"anchor": anchor, "laptop": laptop})
for name, c := range map[string]struct {
raw string
want int
}{
"in output": {`{"ok":true,"output":` + quote(string(export)+"\n") + `}`, 2},
"as answer": {`{"ok":true,"output":"","answer":` + string(export) + `}`, 2},
"refused": {`{"ok":false,"output":"no store"}`, 0},
"no node": {`{"ok":true,"output":"{}"}`, 0},
"no version": {`{"ok":true,"output":"{\"anchor\":{\"node\":\"anchor\",\"modules\":[]}}"}`, 0},
} {
got, err := ExportOf([]byte(c.raw))
if len(got) != c.want || (c.want == 0) != (err != nil) {
t.Errorf("%s: %d nodes, %v", name, len(got), err)
}
}
if seed := SeedChanges(map[string]delivery.Placement{"laptop": laptop, "anchor": anchor}); len(seed) != 2 || seed[0].Node != "anchor" || seed[0].Op != delivery.OpSeed {
t.Errorf("the seed's changes = %+v", seed)
}
}
func quote(s string) string {
raw, _ := json.Marshal(s)
return string(raw)
}
// Issue 774: every call the assigner makes of the controller carries only the arguments the controller's seat contract
// declares for its verb, and every one it requires; the controller refuses any other argument and does nothing. A "why"
// added to every call (2026-10-28) had each approved placement refused.
func TestControllerCallsFollowTheContract(t *testing.T) {
declared := map[string]map[string]any{}
required := map[string][]string{}
for _, v := range catalogue.ControllerVerbs {
props, _ := v.Input["properties"].(map[string]any)
declared[v.Name] = props
switch r := v.Input["required"].(type) {
case []string:
required[v.Name] = r
}
}
db := map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
moved := map[string]delivery.Provider{"db": {Node: "laptop", Module: "store"}}
placed := func(node string, pins map[string]delivery.Provider, modules ...string) delivery.Placement {
p := delivery.Placement{Node: node, Pins: pins}
for _, m := range modules {
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
}
return p
}
ptr := func(p delivery.Placement) *delivery.Placement { return &p }
for name, c := range map[string]struct {
before map[string]delivery.Placement
after map[string]*delivery.Placement
verbs []string
}{
"assign": {map[string]delivery.Placement{}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil, "web"))}, []string{"assign"}},
"unassign": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil))}, []string{"unassign"}},
"key deleted": {map[string]delivery.Placement{"anchor": placed("anchor", db, "web")}, map[string]*delivery.Placement{"anchor": nil}, []string{"unassign", "unpin"}},
"pin set": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", db, "web"))}, []string{"pin"}},
"pin changed": {map[string]delivery.Placement{"anchor": placed("anchor", db, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", moved, "web"))}, []string{"pin"}},
"moved and pinned": {map[string]delivery.Placement{"anchor": placed("anchor", nil, "web")}, map[string]*delivery.Placement{"anchor": ptr(placed("anchor", nil)), "laptop": ptr(placed("laptop", db, "web"))}, []string{"unassign", "assign", "pin"}},
} {
calls := ControllerCalls(c.before, c.after)
var verbs []string
for _, call := range calls {
verbs = append(verbs, call.Verb)
props, ok := declared[call.Verb]
if !ok {
t.Errorf("%s: the controller's seat has no verb %q", name, call.Verb)
continue
}
for arg := range call.Args {
if _, ok := props[arg]; !ok {
t.Errorf("%s: %s gives %q, which mesh-controller.%s does not take, so the controller refuses it", name, call.Key(), arg, call.Verb)
}
}
for _, arg := range required[call.Verb] {
if _, ok := call.Args[arg]; !ok {
t.Errorf("%s: %s lacks %q, which mesh-controller.%s requires", name, call.Key(), arg, call.Verb)
}
}
}
if !slices.Equal(verbs, c.verbs) {
t.Errorf("%s: the calls = %v, want %v", name, verbs, c.verbs)
}
}
}
// Issue 774: a refused call is a condition keyed by its pair, a module's place apart from a pin, naming the verb, the node, the
// module and the refusal; its clear says why.
func TestCallRefusedCondition(t *testing.T) {
assign := ControllerCall{Verb: "assign", Node: "anchor", Module: "web"}
pin := ControllerCall{Verb: "pin", Node: "anchor", Module: "web"}
c := CallRefusedCondition(assign, `assign takes no argument "why"`, true, "")
if c.State != asks.StateRaised || c.Severity != asks.SeverityUrgent || c.Key != "mesh-delivery-assigner.controller-call.anchor.web" ||
!strings.Contains(c.Headline, "assign web on anchor") || !strings.Contains(c.Explanation, `takes no argument "why"`) {
t.Errorf("the raised condition = %+v", c)
}
if CallConditionKey(pin) == c.Key {
t.Error("a pin's condition shares the key of a module's place")
}
if cl := CallRefusedCondition(assign, "", false, "answered"); cl.State != asks.StateCleared || cl.Key != c.Key || cl.Resolved != "answered" {
t.Errorf("the clear = %+v", cl)
}
if _, _, raised, err := FindingOf(c, time.Unix(0, 0)); err != nil || !raised {
t.Errorf("the condition is no finding: %v", err)
}
}
@@ -2,7 +2,6 @@ package assign
import (
"fmt"
"maps"
"slices"
"sort"
"strings"
@@ -10,7 +9,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
@@ -30,15 +28,7 @@ type Module struct {
// 1945, live on 2026-10-25 at 20:04: ace's mesh-build-go was wanted with no account on the bus). Nil: the grant alone.
GrantedOn func(node string) bool
Manifest catalogue.Manifest
// Provisions is the seat mesh-provisioner's provisions this module consumes on a node, by key (ADR 0570 §4): the want
// waits until each is provided. Nil: the seat's state is not read (not delivered yet), and nothing waits on it.
Provisions func(node string) map[string]provisions.Provision
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
Answering func(provision string) []catalogue.Manifest
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
Chosen *catalogue.PersonPolicy
Nodes []string
Nodes []string
// Composed answers the controller's composition of the module for a node at a fingerprint (mesh-controller_composed): its
// digest and the judgement of the move to it, and whether it exists. Nil: none exists (ADR 0539 §1, §6).
Composed func(node, fp string) (delivery.ComposedDigest, bool)
@@ -56,14 +46,8 @@ type Module struct {
Now time.Time
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
Puts map[string]delivery.Put
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
Self func(node string) bool
}
func (m Module) selfOn(node string) bool { return m.Self != nil && m.Self(node) }
func (m Module) composition(node, fp string) (delivery.ComposedDigest, bool) {
if m.Composed == nil {
return delivery.ComposedDigest{}, false
@@ -98,15 +82,14 @@ func Canary(nodes []string, online map[string]bool) string {
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
// down and one module never blocks another.
// limit (ADR 0528 clause 3).
func Holds(m Module, node string, n Nodes) (bool, int, string) {
return holds(m, node, n, true)
}
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
// without which there is nothing to want: the build done, not held, the grant and the composition.
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
// composition.
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
@@ -122,6 +105,11 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return false, delivery.RuleProviders, why
}
}
}
if !m.Granted {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
@@ -133,56 +121,9 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
if _, ok := m.composition(node, m.FP); !ok {
return false, delivery.RuleComposed, fmt.Sprintf("the controller has not composed %s for %s at %s", m.Name, node, short(m.FP))
}
if words, waiting := m.provisionWaiting(node); waiting {
return false, delivery.RuleProvisions, words
}
return true, 0, ""
}
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
func (m Module) provisionWaiting(node string) (string, bool) {
if m.Provisions == nil {
return "", false
}
if missing := m.MissingProvisions(node); len(missing) > 0 {
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
}
keyed := m.Provisions(node)
for _, key := range slices.Sorted(maps.Keys(keyed)) {
if p := keyed[key]; p.State != provisions.Provided {
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
if p.Why != "" {
words += ": " + p.Why
}
return words, true
}
}
return "", false
}
// MissingProvisions are the provisions a want of the module waits for (catalogue.Awaited, the one rule the plan
// composition and the seat mesh-provisioner key by, issue 774) that have no key on node yet, sorted; none while the
// seat's state is not read. A requirement the composition keys nothing for (a toolchain on the module's own machine, a
// login the controller still makes for a module not handed) is not waited for: on 2026-10-30 every new build of
// grafana, dnsmasq, mesh-knowledge, mesh-mail and the builders waited here for a key nobody would make.
func (m Module) MissingProvisions(node string) []string {
if m.Provisions == nil || m.Answering == nil {
return nil
}
have := map[string]bool{}
for _, p := range m.Provisions(node) {
have[p.Provision] = true
}
var out []string
for _, r := range catalogue.Awaited(m.Manifest, m.Answering) {
if !have[r] {
out = append(out, r)
}
}
return out
}
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
// offers it) and holds.
@@ -209,17 +150,6 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
}
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
func unreached(m Module, node string, n Nodes) string {
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return why
}
}
return ""
}
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
// valuable or rebuildable; cache and none do not).
func keepsData(m catalogue.Manifest) bool {
@@ -239,9 +169,6 @@ type Outcome struct {
// Lifted is the hold of m.FP lifted by itself: every node that runs the module runs m.FP healthy (issue 1906). The held
// record and its ask are taken away, and the wants move by the rules again in this pass.
Lifted bool
// NodeFailed is, by node, a failed health check of m.FP on a node while another node runs m.FP healthy: the failure is
// that node's, so m.FP is not held and nothing is set back (ADR 0557 §1, issue 1496). The pass raises its condition once.
NodeFailed map[string]NodeFailure
// Record is the moves a person must allow before the want names them: each is asked once (ADR 0539 §14).
Record []RecordNeed
// Restore is the nodes of a module holding data whose held fingerprint failed its health check there and that are not set
@@ -249,9 +176,6 @@ type Outcome struct {
Restore []string
// Rules is, by node, the rule holding the node back from m.FP and why.
Rules map[string]Holding
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
Unreached map[string]string
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
@@ -268,27 +192,10 @@ type RecordNeed struct {
Words string
}
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
// running an old-form build, got no want and the pass said nothing).
// Holding is a rule holding a node back, and why.
type Holding struct {
Rule int
Words string
Bus bool
}
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
func (h Holding) Reason() string {
if h.Bus {
return h.Words
}
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
func BusBarWords(node, prev, fp string) string {
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
}
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
@@ -304,25 +211,11 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
if !held && !out.Lifted && !m.Released && m.FP != "" {
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
h, ok := n.Haves[node][m.Name]
if !ok || h.FP != m.FP || !failed(h) {
continue
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && failed(h) {
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
held = true
break
}
if others := healthyElsewhere(m, n, node); len(others) > 0 && !failedElsewhere(m, n, node) {
// F runs healthy on another node, so its failure here is this node's, not F's: F is not held, no node is set back,
// and the failing node keeps its want (ADR 0557 §1; issue 1496: on ace a newly assigned mesh-build-node failed its
// first check and the hold moved shanks, g14 and novox, healthy on F for an hour, back). The pass raises the node's
// condition. With no online healthy witness, or with F failed on a second node, the hold and the set-back stand (ADR 0557
// §2, ADR 0429 §6).
if out.NodeFailed == nil {
out.NodeFailed = map[string]NodeFailure{}
}
out.NodeFailed[node] = NodeFailure{Others: others, Words: HeldWords(h, time.Time{})} // said once: no clock in it
continue
}
out.Held = &delivery.Held{Node: node, Words: HeldWords(h, m.Now)}
held = true
break
}
}
mm := m
@@ -367,41 +260,27 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
ok, rule, words := Holds(mm, node, n)
target := m.FP
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == delivery.RuleCanary {
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
if older {
_, composed := m.composition(node, have.FP)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
}
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
switch {
case older:
ok, target = true, have.FP
case oldForm:
// held back by rule 3 as written: on neither list until the canary runs F healthy
default:
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
// order the move from it as from any want.
switch _, composed := m.composition(node, have.FP); {
case !ran:
case have.FP == m.FP:
ok, rule, words = holds(mm, node, n, false)
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
ok, target = true, have.FP
}
}
if why := unreached(m, node, n); why != "" {
if out.Unreached == nil {
out.Unreached = map[string]string{}
}
out.Unreached[node] = why
}
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
n.Haves[node][m.Name].FP != m.FP {
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
// naming it, and wants already written stay (ADR 0557 §2). A node that runs F with no live want gets its want at F, or
// it would be on neither list and its node-engine would give the module up (issue 1945).
ok, rule, words = false, delivery.RuleNotHeld, FailedAloneWords(m.Name, m.FP, slices.Sorted(maps.Keys(out.NodeFailed)))
}
switch {
case ok:
d, _ := m.composition(node, target)
@@ -424,14 +303,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
out.Rules[node] = h
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
out.Neither[node] = h
}
}
continue
}
@@ -497,10 +368,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
}
}
next, write := m.selfDerived(node, before, had, next, n, &out)
if !write {
continue
}
if !had || !delivery.SameWant(normal(before), normal(next)) {
out.Wants[node] = next
}
@@ -513,43 +380,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
return out
}
// selfDerived is next with its Self derived from the placement now (Module.Self), whichever rule made it, and whether it is
// written. A placed node's want names the build alone (ADR 0547 §2). An unplaced node's want that a rule kept from the want
// written while the node was placed (held at F, held back by rule 3 or 5, a data module asked about) names the controller's
// composition of the same fingerprint, asked as any move of a record module is (ADR 0539 §14): on 2026-10-27, after the
// placement seed's revert, g14's systemd-resolved (held) and forticlient (rule 5), and shanks's forticlient (rule 3), kept
// their Self wants, which their node agents, no longer placed, could not compose (issue 1126). With no such composition, or
// while its ask waits, the want written stays and the node's holding says why.
func (m Module) selfDerived(node string, before delivery.Want, had bool, next delivery.Want, n Nodes, out *Outcome) (delivery.Want, bool) {
switch {
case next.Removed:
return next, true
case m.selfOn(node):
next.Self, next.Composed = true, ""
return next, true
case !next.Self && next.Composed != "":
return next, true
}
next.Self, next.Yes = false, ""
d, there := m.composition(node, next.FP)
if !there || d.Digest == "" {
out.Rules[node] = Holding{Rule: delivery.RuleComposed, Words: fmt.Sprintf("%s on %s was composed by the node while it was placed, "+
"and the controller has not composed it at %s, so its want stays", m.Name, node, short(next.FP))}
return next, false
}
at := m
at.FP = next.FP
running, ran := n.Haves[node][m.Name]
if need, wait := at.mustAsk(node, before, had, ran, running.FP, running.Composed == "", d); wait {
if need != nil && !slices.Contains(out.Record, *need) { // the branch held back at an older fingerprint may have asked it
out.Record = append(out.Record, *need)
}
return next, false
}
next.Composed = d.Digest
return next, true
}
// NoWantWords say why a node gets no want for a module: the rule that holds its first want back and why (issue 1945).
func NoWantWords(rule int, words string) string {
return fmt.Sprintf("no want: rule %d: %s", rule, words)
@@ -564,43 +394,6 @@ func failed(h delivery.Have) bool {
return h.Health == delivery.HealthFailed && (h.Phase == delivery.PhaseRunning || h.Phase == delivery.PhaseFailed)
}
// healthyElsewhere is every online node of m but node that runs m.FP healthy, by name.
func healthyElsewhere(m Module, n Nodes, node string) []string {
var out []string
for _, other := range slices.Sorted(slices.Values(m.Nodes)) {
// A witness is online: a node that sleeps keeps the have it last wrote, and a stale have is no evidence that F runs
// healthy now.
if other != node && n.Online[other] && healthy(n.Haves[other][m.Name], m.FP) {
out = append(out, other)
}
}
return out
}
// failedElsewhere says whether another node of m has a have naming m.FP failed: then the failure is not one node's alone, and
// the hold and the set-back stand at the second failure (ADR 0557 §2).
func failedElsewhere(m Module, n Nodes, node string) bool {
for _, other := range m.Nodes {
if h, ok := n.Haves[other][m.Name]; ok && other != node && h.FP == m.FP && failed(h) {
return true
}
}
return false
}
// FailedAloneWords say why a node gets no new want naming fp: fp fails its health check on the nodes named, alone.
func FailedAloneWords(module, fp string, failing []string) string {
return fmt.Sprintf("%s at %s fails its health check on %s, so no further node moves to it until that is over", module, short(fp),
strings.Join(failing, ", "))
}
// NodeFailure is a failed health check of a module's newest fingerprint on one node while it runs healthy on Others: the have's
// health, phase and words (HeldWords without a time), and the healthy witnesses by name.
type NodeFailure struct {
Others []string
Words string
}
// runs says whether a have is of a module the node runs: running, whatever its health.
func runs(h delivery.Have) bool {
return h.Phase == delivery.PhaseRunning
@@ -673,7 +466,12 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
if _, barred := m.busBar(before, had, running); barred {
// The fingerprint the bus ran before: its want's, or with no readable want the one its have names (ADR 0539 §15 bars a new one).
prev := before.FP
if !had {
prev = running
}
if catalogue.ProvidesBus(m.Manifest) && prev != m.FP {
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
}
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
@@ -685,24 +483,9 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
}
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
// readable want the one its have names (ADR 0539 §15 bars a new one).
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
prev = before.FP
if !had {
prev = running
}
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
}
// recorded says whether the module is of policy record: a person's choice at the assigner over everything but the bus (ADR
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
// recorded says whether the module is of policy record: as the controller holds it beside the composition's digest, the
// operator's own choice included (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
if m.Chosen != nil {
policy, _, _, _ := catalogue.Upgrade(m.Manifest, m.Chosen)
return policy == catalogue.PolicyRecord
}
if d.Policy != "" {
return d.Policy == delivery.PolicyRecord
}
@@ -8,7 +8,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
@@ -66,16 +65,13 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
delivery.RuleCanary: func(_ *Module, n *Nodes) {
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
},
delivery.RuleProviders: func(_ *Module, n *Nodes) {
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
},
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
delivery.RuleComposed: func(m *Module, _ *Nodes) {
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
},
// 1954/D1 (ADR 0570 §4): a provision of the module on that node still asked holds its want.
delivery.RuleProvisions: func(m *Module, _ *Nodes) {
m.Provisions = func(node string) map[string]provisions.Provision {
return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Asked}, node + ".mail.oidc": {Provision: "oidc", State: provisions.Provided}}
}
},
} {
m, n := ready()
breakIt(&m, &n)
@@ -89,205 +85,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) {
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
}
if rule == delivery.RuleProvisions && (!strings.Contains(out.Rules["g14"].Words, "g14.mail.database") || !strings.Contains(out.Rules["g14"].Words, "asked")) {
t.Errorf("the provision's hold does not name its key and state: %s", out.Rules["g14"].Words)
}
}
// 1954/D1: every provision provided, the want moves; with the seat's state not read, nothing waits on it.
for _, all := range []func(string) map[string]provisions.Provision{
func(node string) map[string]provisions.Provision {
return map[string]provisions.Provision{node + ".mail.database": {Provision: "database", State: provisions.Provided}}
}, nil} {
m, n := ready()
m.Provisions = all
if out := Wants(m, n, wantedOld(), nil); out.Wants["g14"].FP != fNew {
t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
}
}
// 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named; and only one the
// composition keys (catalogue.Awaited, issue 774), replaying 2026-10-30: grafana (not handed) waited on ace for
// oidc-client, which the controller still makes, and mesh-build-go (handed) for go-toolchain on its own machine.
{
handed := &catalogue.Handed{Source: "s", Digest: "d", Templates: map[string]string{"c": "c"}}
keycloak := catalogue.Manifest{Module: "keycloak", Provides: []catalogue.Offer{{Name: "oidc-client", Scope: catalogue.ScopeMesh}}}
proxy := catalogue.Manifest{Module: "route-proxy", Provides: []catalogue.Offer{{Name: "route", Scope: catalogue.ScopeMesh,
AnsweredBy: catalogue.AnsweredByHelper}}}
store := catalogue.Manifest{Module: "store", Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}}
goTools := catalogue.Manifest{Module: "go", Provides: []catalogue.Offer{{Name: "go-toolchain", Scope: catalogue.ScopeNode}}}
gitea := catalogue.Manifest{Module: "gitea", Provides: []catalogue.Offer{{Name: "forge-account", Scope: catalogue.ScopeMesh}}}
answering := func(p string) []catalogue.Manifest {
for _, m := range []catalogue.Manifest{keycloak, proxy, store, goTools, gitea} {
if m.Provides[0].Name == p {
return []catalogue.Manifest{m}
}
}
return nil
}
for _, c := range []struct {
name string
manifest catalogue.Manifest
missing string // the provision named, "" for none: the want moves
}{
{"a handed module's database with no key", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
Requires: []string{"database"}}, "database"},
{"a module not handed waits for no login the controller makes (grafana on ace, oidc-client)",
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client"}}, ""},
{"a module not handed waits for its route, which the helper answers (grafana's route)",
catalogue.Manifest{Module: "mail", Requires: []string{"oidc-client", "route"}}, "route"},
{"a handed builder waits for no toolchain on its own machine (mesh-build-go, go-toolchain)",
catalogue.Manifest{Module: "mail", Moved: true, Handed: handed, Requires: []string{"go-toolchain"}}, ""},
{"and does wait for its forge account", catalogue.Manifest{Module: "mail", Moved: true, Handed: handed,
Requires: []string{"go-toolchain", "forge-account"}}, "forge-account"},
} {
m, n := ready()
m.Manifest, m.Answering = c.manifest, answering
m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} }
out := Wants(m, n, wantedOld(), nil)
if c.missing == "" {
if out.Wants["g14"].FP != fNew || len(m.MissingProvisions("g14")) != 0 {
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
}
continue
}
if out.Rules["g14"].Rule != delivery.RuleProvisions || !strings.Contains(out.Rules["g14"].Words,
"provision "+c.missing+" has no key yet") || len(m.MissingProvisions("g14")) != 1 {
t.Errorf("%s: %+v", c.name, out.Rules["g14"])
}
}
}
// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
// healthy held photos, and novox offline left a module with no canary.
for _, c := range []struct {
name, module, provision, provider string
have *delivery.Have // what novox runs of the module; nil: a new assignment
before map[string]delivery.Want
online bool
want string
}{
{"de-spiegel, route-proxy down, old form", "de-spiegel", "route", "route-proxy",
&delivery.Have{FP: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}, nil, true, fNew},
{"link2pay, route-proxy down, new assignment", "link2pay", "route", "route-proxy", nil, nil, true, fNew},
{"photos, minio not healthy, moving its want", "photos", "s3-bucket", "minio",
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, true, fNew},
{"novox offline, no canary, new assignment", "link2pay", "route", "route-proxy", nil, nil, false, fNew},
{"novox offline, no canary, moving its want", "photos", "s3-bucket", "minio",
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, false, fOld},
} {
m := Module{Name: c.module, FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: c.module, Requires: []string{c.provision}}}
n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]delivery.Have{"novox": {
c.provider: {FP: fpOf('p'), Phase: delivery.PhaseFailed, Health: delivery.HealthFailed}}},
Providers: map[string]map[string][]string{c.provision: {c.provider: {"novox"}}}}
if c.have != nil {
n.Haves["novox"][c.module] = *c.have
}
out := Wants(m, n, c.before, nil)
w, written := out.Wants["novox"]
if !written {
w = c.before["novox"]
}
if w.FP != c.want || w.Removed {
t.Errorf("%s: novox's want = %+v, want %s", c.name, w, short(c.want))
}
if _, on := out.Neither["novox"]; on {
t.Errorf("%s: on neither list: %+v", c.name, out.Neither)
}
if c.online && !strings.Contains(out.Unreached["novox"], c.provision) {
t.Errorf("%s: the provider down is not said: %q", c.name, out.Unreached["novox"])
}
}
// 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the placement is gone, names
// the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient,
// rule 5, kept their Self wants, which g14 could no longer compose); with no such composition, or while a record ask waits, it
// stays and is said; a placed node's want is Self with no composition.
selfAt := func(fp string) map[string]delivery.Want {
return map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fp, Self: true, Yes: dig(fp)}}
}
atF := func(_ *Module, n *Nodes) {
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Composed: dig(fNew), Phase: delivery.PhaseFailed, Health: delivery.HealthOK}
}
for _, c := range []struct {
name string
set func(*Module, *Nodes)
before string
want delivery.Want // the want of g14 written; zero: none written
rule int // the rule g14's holding names
asked bool
}{
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}},
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleCanary},
{name: "rule 3 held back at an older fingerprint", before: fOld, set: func(_ *Module, n *Nodes) {
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
}, want: delivery.Want{FP: fOld, Composed: dig(fOld), Newest: fNew, Rule: delivery.RuleCanary,
Words: "the canary ace does not run bbbbbbbbbbbb healthy yet"}, rule: delivery.RuleCanary},
{name: "held at F with nothing to set back to", before: fNew, set: func(m *Module, n *Nodes) { atF(m, n); m.Held = true },
want: delivery.Want{FP: fNew, Composed: dig(fNew), Words: "mail is held at bbbbbbbbbbbb and g14 ran no other fingerprint healthy to set it back to"},
rule: delivery.RuleNotHeld},
{name: "held data module", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Held = true
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew),
Words: "mail holds data, so it is not set back from bbbbbbbbbbbb: the operator is asked whether to restore it"}, rule: delivery.RuleNotHeld},
{name: "a record ask waits", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
recording(m)
m.Held = true
m.Composed = func(_, fp string) (delivery.ComposedDigest, bool) {
return delivery.ComposedDigest{Digest: dig(fp), Judgement: delivery.JudgedRecreates, Against: dig(fOld)}, true
}
}, rule: delivery.RuleNotHeld, asked: true},
{name: "no composition of F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
if node == "g14" {
return delivery.ComposedDigest{}, false
}
return composedEverywhere(node, fp)
}
}, rule: delivery.RuleComposed},
{name: "placed", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
m.Self = func(node string) bool { return node == "g14" }
m.Composed = func(node, fp string) (delivery.ComposedDigest, bool) {
if node == "g14" {
return delivery.ComposedDigest{Judgement: delivery.JudgedNeither}, true
}
return composedEverywhere(node, fp)
}
}, want: delivery.Want{FP: fNew, Self: true}},
} {
m, n := ready()
c.set(&m, &n)
before := selfAt(c.before)
if c.name == "placed" {
before["g14"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
}
out := Wants(m, n, before, nil)
if got := out.Wants["g14"]; got != c.want {
t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want)
}
if got := out.Rules["g14"].Rule; got != c.rule {
t.Errorf("%s: g14 is held by rule %d (%s), want %d", c.name, got, out.Rules["g14"].Words, c.rule)
}
if asked := len(out.Record) == 1 && out.Record[0].Node == "g14"; asked != c.asked {
t.Errorf("%s: record asks %+v", c.name, out.Record)
}
if w, ok := out.Wants["g14"]; ok {
if _, err := delivery.EncodeWant(w); err != nil {
t.Errorf("%s: g14's want does not encode: %v", c.name, err)
}
}
}
// Rule 6 per node: a grant written for the module's other nodes does not grant a node whose account it does not name
@@ -353,72 +150,6 @@ func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
t.Errorf("g14 moved to a held fingerprint: %+v", w)
}
// 1496, ADR 0557 §1, §2: the same failure while F runs healthy on another node is that node's: F is not held, nothing is set
// back, the failing node keeps its want, and the outcome names it (on ace a newly assigned module's first check held a version healthy for an hour on three nodes).
n.Haves["g14"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
before := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fNew)}, "g14": {FP: fNew, Composed: dig(fNew)}}
out = Wants(m, n, before, map[string]string{"ace": fOld, "g14": fOld})
if out.Held != nil || len(out.Wants) != 0 {
t.Fatalf("held %+v, wants %+v: want no hold and no want changed", out.Held, out.Wants)
}
if f := out.NodeFailed["ace"]; !slices.Equal(f.Others, []string{"g14"}) || f.Words != "health failed, phase running: connection refused" ||
len(out.NodeFailed) != 1 {
t.Errorf("node failed = %+v", out.NodeFailed)
}
// ADR 0557 §2 (version 9): while one node fails F alone, novox, whose want names fOld, gets no new want naming F; the wants
// at F stay. The canary ace runs F healthy here and g14 fails it, so the canary rule (3) would let novox move: only §2 holds it.
three := m
three.Nodes = []string{"ace", "g14", "novox"}
failedHave := n.Haves["ace"]["mail"]
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = n.Haves["g14"]["mail"], failedHave
n.Online["novox"] = true
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}
before["novox"] = delivery.Want{FP: fOld, Composed: dig(fOld)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if _, alone := out.NodeFailed["g14"]; !alone {
t.Errorf("g14 does not fail alone: %+v", out.NodeFailed)
}
if w, written := out.Wants["novox"]; out.Held != nil || len(out.NodeFailed) != 1 || (written && w.FP == fNew) || out.Rules["novox"].Rule != delivery.RuleNotHeld {
t.Errorf("held %+v, node failed %+v, novox's want %+v (%+v): want novox kept off F", out.Held, out.NodeFailed, out.Wants["novox"], out.Rules["novox"])
}
if _, written := out.Wants["g14"]; written {
t.Errorf("g14's want at F was rewritten: %+v", out.Wants["g14"])
}
// A node that runs F healthy with no want, while g14 fails it alone, gets its want at F and is on no neither list (issue 1945).
n.Haves["novox"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
delete(before, "novox")
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if w := out.Wants["novox"]; w.FP != fNew || len(out.Neither) != 0 {
t.Errorf("novox running F with no want: want %+v, neither %+v; want its want at F and no neither", w, out.Neither)
}
n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] = failedHave, n.Haves["ace"]["mail"]
// ADR 0557 §2: a witness that is offline keeps the have it last wrote and witnesses nothing (D1); F failing on a second node is
// not one node's failure (D3). Either way F is held and ace is set back.
for _, c := range []struct {
name string
online map[string]bool
novox delivery.Health
}{
{"the witness sleeps", map[string]bool{"ace": true, "g14": false, "novox": true}, delivery.HealthOK},
{"F fails on a second node", map[string]bool{"ace": true, "g14": true, "novox": true}, delivery.HealthFailed},
} {
n.Online = c.online
n.Haves["novox"] = map[string]delivery.Have{"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: c.novox}}
if c.novox == delivery.HealthOK {
n.Haves["novox"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
}
before["novox"] = delivery.Want{FP: fNew, Composed: dig(fNew)}
out = Wants(three, n, before, map[string]string{"ace": fOld, "g14": fOld, "novox": fOld})
if out.Held == nil || out.Held.Node != "ace" || len(out.NodeFailed) != 0 {
t.Errorf("%s: held %+v, node failed %+v: want F held at ace", c.name, out.Held, out.NodeFailed)
}
if w := out.Wants["ace"]; w.FP != fOld {
t.Errorf("%s: ace's want = %+v, want set back", c.name, w)
}
}
}
// 1297/B4: a module holding data is not set back; its want stays and says why.
@@ -696,23 +427,6 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
}
m.Puts = nil
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
// condition stays raised. A node with a live want keeps it and is not on neither list.
m.Composed = composedEverywhere
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
if w, wrote := out.Wants["ace"]; wrote {
t.Errorf("the bus bar wrote ace's want %+v", w)
}
h, on := out.Neither["ace"]
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
!strings.Contains(h.Words, "assigner_put") {
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
}
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
}
stepsOfAPut(t)
}
@@ -874,12 +588,6 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
if out := Wants(m, n, wanted, nil); len(out.Record) == 0 {
t.Errorf("a hand-made composition fell through the manifest's record policy: %+v", out)
}
// 1953/D4 (ADR 0565 §7): a person's choice at the assigner is over the controller's beside the composition.
with(delivery.PolicyRecord, "person")
m.Chosen = &catalogue.PersonPolicy{V: catalogue.PersonPolicyVersion, Module: m.Name, Policy: catalogue.PolicyRoll, Why: "w", By: "op"}
if out := Wants(m, n, wanted, nil); len(out.Record) != 0 || out.Wants["ace"].FP != fNew {
t.Errorf("a person's roll at the assigner was not honoured over the controller's record: %+v", out)
}
}
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
@@ -1081,12 +789,10 @@ func x11() (Module, Nodes) {
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
m, n := x11()
out := Wants(m, n, nil, nil)
// 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said
if w := out.Wants["g14"]; w.FP != fNew || out.Unreached["g14"] == "" {
t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"])
}
if w := out.Wants["shanks"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleCanary {
t.Errorf("shanks's want = %+v, want its first want at the fingerprint it runs, held back from F by the canary", w)
for _, node := range []string{"g14", "shanks"} {
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
}
}
if len(out.Neither) != 0 {
t.Errorf("left on neither list: %+v", out.Neither)
@@ -1124,22 +830,11 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
if what == "an old-form build" {
// the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("%s: shanks got a want: %+v", what, w)
}
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
t.Errorf("%s: shanks on neither list = %+v (named %v), want rule 3", what, h, ok)
}
continue
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("%s: shanks got a want: %+v", what, w)
}
// 1960: no want can name what it runs, so its first want names F rather than none: never on neither list
if w := out.Wants["shanks"]; w.FP != fNew {
t.Errorf("%s: shanks's want = %+v, want F", what, w)
}
if h, ok := out.Neither["shanks"]; ok {
t.Errorf("%s: shanks on neither list = %+v", what, h)
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
}
}
m, n := x11()
@@ -1148,12 +843,10 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
if len(out.Neither) != 0 {
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
}
// the canary g14 moves to F (its provider's health no rule); shanks waits for it
if w := out.Wants["shanks"]; w.FP != "" && w.FP != fOld {
t.Errorf("shanks's want moved from its want past the canary: %+v", w)
}
if w := out.Wants["g14"]; w.FP != fNew {
t.Errorf("the canary g14's want = %+v, want F", w)
for node, w := range out.Wants {
if w.FP != fOld {
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
}
}
}
@@ -1164,12 +857,11 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
m, n := x11()
delete(n.Haves["shanks"], "screen-lock")
out := Wants(m, n, nil, nil)
// 1960: the canary orders moves, never a first want: the new assignment is wanted at F
if w := out.Wants["shanks"]; w.FP != fNew {
t.Errorf("a new assignment while the canary does not run F: %+v, want F", w)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
}
if _, ok := out.Neither["shanks"]; ok {
t.Errorf("a new assignment is named on neither list: %+v", out.Neither)
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
}
m.Built = false
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
@@ -1211,7 +903,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
// stays on neither list; the providers hold no want (issue 1960). The old-form node that is itself the canary gets F once its providers run healthy.
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
oldForm := func(n Nodes, node string) {
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
@@ -1245,16 +937,16 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
}
// 1960: the canary healthy on F but the providers not healthy on shanks: F all the same, the provider down said
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
m, n = x11()
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, nil, nil)
if w := out.Wants["shanks"]; w.FP != fNew || out.Unreached["shanks"] == "" {
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("providers not healthy: shanks got a want: %+v", w)
}
if _, on := out.Neither["shanks"]; on {
t.Errorf("providers not healthy: shanks on neither list: %+v", out.Neither)
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
}
// F not granted: nothing, even with the canary healthy
m, n = x11()
+11 -34
View File
@@ -7,7 +7,6 @@ import (
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func load(t *testing.T) catalogue.Manifest {
@@ -16,9 +15,9 @@ func load(t *testing.T) catalogue.Manifest {
if err != nil {
t.Fatal(err)
}
m, err := catalogue.Decode(raw)
m, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatalf("mesh-sdk cannot decode the manifest: %v", err)
t.Fatalf("mesh-sdk's manifest reader refuses the manifest: %v", err)
}
return m
}
@@ -33,9 +32,9 @@ func TestManifest(t *testing.T) {
}
reads := slices.Clone(m.Reads)
slices.Sort(reads)
want := []string{"mesh-build-go.steps", "mesh-conditions.takeover", "mesh-config.kept", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease",
"mesh-delivery-builder.build-log", "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-provisioner.provisions", "node-engine.have"}
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
@@ -43,8 +42,7 @@ func TestManifest(t *testing.T) {
for _, s := range m.State {
states[s.Name] = s.History
}
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20,
"placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5}; !reflect.DeepEqual(states, want) {
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -59,33 +57,12 @@ func TestManifest(t *testing.T) {
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
}
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
// module), says each put's step as the event put, and calls the controller's warranted and the bus node's backup holder.
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its seven verbs; keeps the seat's events on its stream
// placement for 30 days; calls the controller's export, its four verbs that keep its assignments in step and its
// hand-act log; and reads mesh-config's resolved values to compose every node a proposal touches.
// ADR 0565 §7, §9 (issue 1953 D4): and the seat mesh-wants, serving upgrade and bus.
if len(m.Claims) != 2 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh ||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) ||
m.Claims[1].Name != catalogue.WantsSeat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus"}) {
t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs and mesh-wants with its two", m.Claims)
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
// calls the controller's warranted and the bus node's backup holder.
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
}
if seat, _ := catalogue.SeatNamed(catalogue.WantsSeat); catalogue.CanHold(m, seat) != nil {
t.Errorf("the assigner cannot hold mesh-wants: %v", catalogue.CanHold(m, seat))
}
if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil {
t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat))
}
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
t.Errorf("the assigner serves tools %v, want its three put tools", m.Tools)
}
if len(m.Streams) != 1 || m.Streams[0].Name != delivery.PlacementStream || m.Streams[0].MaxAgeDays != delivery.PlacementStreamDays ||
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
}
if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.hand-act", "seat:mesh-controller.pin",
"seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin",
"seat:mesh-controller.warranted", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat, the controller's only while it has no holder
"seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
if !slices.Equal(m.Emits, []string{"put"}) {
+4 -61
View File
@@ -2,47 +2,18 @@
"module": "mesh-delivery-assigner",
"version": "1",
"slug": "assigner",
"claims": [
{
"name": "mesh-placement",
"scope": "mesh",
"serves": [
"placement",
"proposals",
"propose",
"withdraw",
"again",
"adopt",
"hand-back"
]
},
{
"name": "mesh-wants",
"scope": "mesh",
"serves": [
"upgrade",
"bus"
]
}
],
"reads": [
"mesh-build-go.steps",
"mesh-conditions.takeover",
"mesh-config.kept",
"mesh-config.resolved",
"mesh-controller.assignments",
"mesh-controller.builds",
"mesh-controller.composed",
"mesh-controller.grants",
"mesh-controller.lease",
"mesh-delivery-builder.build-log",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-teller.answers",
"mesh-delivery-watchman.presence",
"mesh-grants.grants",
"mesh-provisioner.provisions",
"node-engine.have"
],
"uses": [
@@ -50,18 +21,9 @@
"operator-channel"
],
"invokes": [
"seat:mesh-controller.assign",
"seat:mesh-controller.hand-act",
"seat:mesh-controller.pin",
"seat:mesh-controller.placement-export",
"seat:mesh-controller.unassign",
"seat:mesh-controller.unpin",
"seat:mesh-controller.warranted",
"seat:mesh-hand-acts.hand-act",
"seat:mesh-hand-acts.hand-acts",
"seat:mesh-hand-acts.warranted",
"seat:node-backup.backed-up",
"seat:node-backup.now"
"seat:node-backup.now",
"seat:node-backup.backed-up"
],
"emits": [
"put"
@@ -72,30 +34,11 @@
"assigner_puts"
],
"state": [
{ "name": "placement", "history": 64 },
{ "name": "proposals", "history": 5 },
{ "name": "candidates", "history": 5 },
{ "name": "pulses", "key-ttl": true },
{ "name": "asks", "history": 5 },
{ "name": "compositions", "history": 5 },
{ "name": "differences", "history": 64 },
{ "name": "puts", "history": 20 },
{ "name": "wants", "history": 20 },
{ "name": "findings", "history": 5 },
{ "name": "policies", "history": 20 },
{ "name": "seat-holders", "history": 5 }
],
"streams": [
{
"name": "placement",
"events": [
"placement-proposed",
"placement-judged",
"placement-changed",
"placement-withdrawn"
],
"max-age-days": 30
}
{ "name": "wants", "history": 20 }
],
"own-secrets": {
"broker": "${dir:state}/broker"
@@ -7,7 +7,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
@@ -16,7 +15,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-builder/internal/build"
)
@@ -25,16 +23,15 @@ import (
type queueAsker func(key string, value []byte) (json.RawMessage, error)
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, since build.LogSince, now func() time.Time) []stdio.Tool {
// the builder's read.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: build.StuckTool,
Description: "Every build key that waits (building or queued), oldest first: its module, fingerprint, step words and since when, and each ask it " +
"waits on by language with its id, when it was put, its asking (1 for the first; the builder asks an ask again when it can no longer be " +
"answered, at most 3 times) and what its build module's steps hold of it (none, or its state on its node since its start). The " +
"seat workers' delivery counts are not readable by the builder. dead_letters says whether the builder hears the asks the build seats give up " +
"on (issue 734), and why not: heard, a given-up ask is asked again at once; deaf, at its check bound. With module, that module's keys only. Read-only.",
"seat workers' delivery counts are not readable by the builder. With module, that module's keys only. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "only this module's keys"}}},
Run: func(args map[string]any) (any, error) { return r.Stuck(str(args, "module")) }},
@@ -58,84 +55,6 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
return map[string]any{"key": key, "asked": true,
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
}},
{Name: build.BuildAgainTool,
Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name"},
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
"why": map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
raw, err := buildAgain(key, value)
if err != nil {
return nil, err
}
r, err := build.DecodeReAsk(raw)
if err != nil {
return nil, err
}
if r.Refused {
return nil, errors.New(r.Answer)
}
if r.Failed != "" {
return nil, errors.New(r.Answer)
}
if r.Revision == 0 && !r.Pending {
return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
}
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
}},
{Name: delivery.BuildsSinceTool,
Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " +
"log held more than one answer carries: ask again from the last build's time. What mesh-catalog registers a build from. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
Run: func(args map[string]any) (any, error) {
at, err := time.Parse(time.RFC3339Nano, str(args, "since"))
if err != nil {
return nil, fmt.Errorf("since %q is not an RFC 3339 time: %w", str(args, "since"), err)
}
return build.BuildsSince(since, at)
}},
{Name: build.AnnounceAgainTool,
Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name"},
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
"why": map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
raw, err := buildAgain(key, value)
if err != nil {
return nil, err
}
r, err := build.DecodeReAsk(raw)
if err != nil {
return nil, err
}
if r.Refused || r.Revision == 0 {
return nil, errors.New(r.Answer)
}
return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
}},
{Name: build.SeatQueueTool,
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
@@ -166,17 +85,8 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
a, err := build.DecodeQueueAsk(v)
return err == nil && !a.Answered.IsZero()
})
}
// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
// bound for a value written after it that answered says is the builder's answer.
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
return func(key string, value []byte) (json.RawMessage, error) {
heard := make(chan json.RawMessage, 1)
answered := make(chan json.RawMessage, 1)
var rev uint64
var mu sync.Mutex
if err := s.Watch(key, func(c stdio.StateChange) error {
@@ -186,34 +96,36 @@ func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration,
if c.Op != "put" || mine == 0 || c.Revision <= mine {
return nil
}
if answered(c.Value) {
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
select {
case heard <- c.Value:
case answered <- c.Value:
default:
}
}
return nil
}); err != nil {
return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
}
mu.Lock()
rev = r
mu.Unlock()
// The watch may have heard the answer before the revision was known: read it once too.
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
return e.Value, nil
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
return e.Value, nil
}
}
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
case v := <-heard:
case v := <-answered:
return v, nil
case <-wait.Done():
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
}
}
}
@@ -254,17 +166,12 @@ func main() {
steps[bm] = read(stdio.State(bm + ".steps"))
}
reasks := stdio.State(build.ReAsksState)
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps,
Hearing: read(stdio.State(buildrun.HearingState))}
r := build.StuckReader{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] "+f+"\n", a...) }, Builds: read(stdio.State("builds")), ReAsks: read(reasks), Steps: steps}
put := func(key string, value []byte) error {
_, err := reasks.Put(key, json.RawMessage(value))
return err
}
buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
a, err := build.DecodeReAsk(v)
return err == nil && !a.Answered.IsZero()
})
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now)); err != nil {
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
os.Exit(1)
}
@@ -123,6 +123,5 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
if err != nil {
return catalogue.Manifest{}, err
}
// Decoded, never judged (novox/hq ADR 0578 §3): what a build needs of it is read, a field it does not know set aside.
return catalogue.Decode([]byte(kept.JSON))
return catalogue.ParseManifestForInputs([]byte(kept.JSON))
}
@@ -23,9 +23,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-builder/internal/build"
)
@@ -269,12 +267,6 @@ func startBedWith(t *testing.T, closed bool) *bed {
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
b.instances = mk(instancesBucket, 1)
// The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects.
logStream := buscompose.LogStreamName(delivery.BuilderModule, delivery.BuilderLog)
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"},
AllowDirect: true}); err != nil {
t.Fatal(err)
}
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
b.fake.closed.Store(closed)
b.fake.serve(t, "mesh-build-go", "n1", "n2")
@@ -287,29 +279,6 @@ func startBedWith(t *testing.T, closed bool) *bed {
return b
}
// logged is every entry of module's key in the builder's log, read through process.Conn.ReadLog on the controller's user.
func (b *bed) logged(t *testing.T, module string) []delivery.BuildLogEntry {
t.Helper()
conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"})
if err != nil {
t.Fatal(err)
}
defer conn.Close()
entries, err := conn.ReadLog(delivery.BuilderModule, delivery.BuilderLog).Since(time.Unix(0, 0), module)
if err != nil {
t.Fatal(err)
}
var out []delivery.BuildLogEntry
for _, le := range entries {
e, err := delivery.DecodeBuildLogEntry(le.Value)
if err != nil {
t.Fatal(err)
}
out = append(out, e)
}
return out
}
// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does.
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
t.Helper()
@@ -376,7 +345,7 @@ func (b *bed) archive(t *testing.T, name, path, at string) string {
func (b *bed) start(t *testing.T, node string, env map[string]string) {
t.Helper()
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
@@ -436,13 +405,6 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" {
t.Errorf("toolchains = %+v", v.Toolchains)
}
// 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a
// reader with the log's read reads them, the end naming the manifest and what was made.
ended := b.logged(t, "netcheck")
if len(ended) != 2 || ended[0].Event != delivery.BuildStartedEvent || ended[1].Event != delivery.BuildEndedEvent ||
ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" {
t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended)
}
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} {
if got := b.fake.count(tool); got != want {
t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want)
@@ -6,6 +6,12 @@ package main
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
@@ -45,18 +51,80 @@ func main() {
}
}
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
func readCredential(path string) (credential, error) {
raw, err := os.ReadFile(path)
if err != nil {
return credential{}, err
}
var c credential
if err := json.Unmarshal(raw, &c); err != nil {
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
}
if c.URL == "" || c.User == "" {
return credential{}, errors.New("the broker credential names no bus address or user")
}
return c, nil
}
// pinned accepts exactly the certificate with this SHA-256 and no other.
func pinned(want string) *tls.Config {
return &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
MinVersion: tls.VersionTLS12,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("the bus presented no certificate")
}
sum := sha256.Sum256(rawCerts[0])
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
}
return nil
},
}
}
// run is the program: it returns when ctx ends or the bus cannot be used.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-builder: ", log.LstdFlags)
cred, err := process.ReadBrokerAccount(getenv)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
}
cred, err := readCredential(path)
if err != nil {
return err
}
conn, js, err := cred.Dial()
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
nats.CustomInboxPrefix("_INBOX." + cred.User),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return err
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
builds, err := process.EssentialBucket(ctx, js, "builder", delivery.BuilderBuildsBucket, "claim or write a build key")
if err != nil {
return err
@@ -81,37 +149,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
asks := buildrun.JetStreamAsks{JS: js}
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the wait for its step at once,
// and the lane asks its part again (Builder.waitOnce) instead of waiting for the check bound.
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
// state hearing, which builder_stuck says (dead_letters).
dead := &buildrun.DeadLetters{}
// The state may not be on the bus yet when a delivery started the builder (the grants part creates it, issue 1790): it
// is bound when the bus says it was created, and the latest hearing is written then.
hearingKV := process.AwaitBucket(ctx, js, conn, delivery.BuilderModule+"_"+buildrun.HearingState, "builder_stuck cannot say whether DEAD_LETTERS is heard", logger.Printf)
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
defer hearing.Stop()
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
var buildModules []string
for _, l := range delivery.AskLanguages {
if m, err := delivery.BuildModuleOf(l); err == nil {
buildModules = append(buildModules, m)
}
}
heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf)
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
account, err := process.ReadAccount(path)
if err != nil {
return err
}
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
}
defer calls.Close()
b := &build.Builder{
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound},
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
Held: heldOnTheBus{js},
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
Gone: heldOnTheBus{js},
@@ -120,7 +171,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Manifests: manifestReader{manifests},
Logf: logger.Printf,
Stored: storedVia(calls.Call),
Node: cred.Node,
}
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
@@ -130,13 +180,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: b.TrunkMoved, errs: errs, logf: logger.Printf}
trunks.start()
b.Trunk = trunks.head
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
// indexer and the assigner through their reads, and by mesh-catalog through the tool builder_builds_since.
b.Log = func(e delivery.BuildLogEntry) {
if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil {
logger.Printf("%v", err)
}
}
b.Said = func(e build.AskedAgain) {
if err := emitAskedAgain(context.WithoutCancel(ctx), js, cred, e); err != nil {
logger.Printf("%v", err)
@@ -161,11 +204,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// The toolchains first, so a build that starts from the first modules value finds the nodes.
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
// placement is settled first, so the toolchains settle through the gate already knowing it.
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
watch.put = gate.Toolchain
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, b.NodesChanged, logger.Printf)
assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger)
if err != nil {
return err
}
@@ -177,12 +220,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// module.
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
stream := toolchainStreams[module]
cons, err := cred.EventConsumer(ctx, js, stream)
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
if err != nil {
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the builder starts again: %v", cred.Node, cred.Module, stream, module, err)
continue
}
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { watch.Reported(m) }, logger.Printf)
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
@@ -191,12 +235,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
// so; it still asks held at its start and at every module value.
switch cons, err := cred.EventConsumer(ctx, js, eventsStream); {
switch cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module); {
case err != nil:
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s is not heard until the builder starts again: %v",
cred.Node, cred.Module, eventsStream, delivery.ArtifactsGoneEvent, err)
default:
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) }, logger.Printf)
cc, err := cons.Consume(func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
@@ -235,25 +280,24 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// A person asks a key's build again with builder_ask_again, which writes the reasks state (issue 1945): watched, as
// every other input. The builder asks again by itself without it.
// A state not on the bus yet at this start is watched once it appears (issue 1790).
process.AwaitBucket(ctx, js, conn, build.ReAsksBucket, "builder_ask_again is not heard; an ask that can no longer be answered is still asked again by itself", logger.Printf).Ready(func(kv jetstream.KeyValue) {
if kv := process.OptionalBucket(ctx, js, build.ReAsksBucket, "builder_ask_again is not heard until the builder starts again; an ask that can no longer be answered is still asked again by itself", logger.Printf); kv != nil {
eachReAsk := func(e jetstream.KeyValueEntry) { reAsked(ctx, b, kv, e, logger.Printf) }
if rw, err := settle(ctx, kv, eachReAsk); err != nil {
process.NotWatched(logger.Printf, build.ReAsksBucket, err, "builder_ask_again is not heard until the builder starts again")
} else {
go func() { errs <- follow(ctx, rw, eachReAsk) }()
}
})
}
// A person reads a build seat's stream with builder_seat_queue, which writes the queues state (issue 1945 note 62): the
// builder alone may read the stream, so it answers there.
process.AwaitBucket(ctx, js, conn, build.QueuesBucket, "builder_seat_queue is not answered", logger.Printf).Ready(func(kv jetstream.KeyValue) {
if kv := process.OptionalBucket(ctx, js, build.QueuesBucket, "builder_seat_queue is not answered until the builder starts again", logger.Printf); kv != nil {
eachRead := func(e jetstream.KeyValueEntry) { queueRead(ctx, heldOnTheBus{js}, kv, e, logger.Printf) }
if qw, err := settle(ctx, kv, eachRead); err != nil {
process.NotWatched(logger.Printf, build.QueuesBucket, err, "builder_seat_queue is not answered until the builder starts again")
} else {
go func() { errs <- follow(ctx, qw, eachRead) }()
}
})
}
logger.Printf("started on %s", cred.Node)
go func() { errs <- follow(ctx, w, each) }()
select {
@@ -262,7 +306,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
case err := <-errs:
return err
case err := <-placementErrs:
case err := <-assignErrs:
return err
case err := <-watch.errs:
return err
@@ -8,9 +8,6 @@ import (
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
@@ -20,14 +17,8 @@ import (
// askAgainer is the part of the builder a reasks value asks through.
type askAgainer interface {
AskAgain(key, why string) []string
BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult
}
// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
// longer for the builder's answer.
const buildAgainBound = 15 * time.Second
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
// answered already (this builder's own write) or deleted is left.
@@ -43,15 +34,6 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
if r.Answer != "" {
return
}
if r.Mode == build.ModeBuild {
// The answer comes when the claim and the asks are written, so it is made beside the watch, not in it.
go builtAgain(ctx, b, kv, e, r, logf)
return
}
if r.Mode == build.ModeAnnounce {
announcedAgain(ctx, b, kv, e, r, logf)
return
}
asked := b.AskAgain(e.Key(), r.Why)
if len(asked) == 0 {
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
@@ -71,12 +53,12 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
// emitAskedAgain says the builder's event build.asked-again (issue 1945): mesh.mod.<module>.event.build.asked-again,
// with the SDK's envelope headers and the new ask's id as the bus's dedup id.
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Account, e build.AskedAgain) error {
func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred credential, e build.AskedAgain) error {
data, err := json.Marshal(e)
if err != nil {
return err
}
msg := nats.NewMsg(cred.EventSubject(build.AskedAgainEvent))
msg := nats.NewMsg("mesh.mod." + cred.Module + ".event." + build.AskedAgainEvent)
id := "asked-again-" + e.New
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
@@ -91,75 +73,3 @@ func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Ac
}
return nil
}
// builtAgain answers a reasks value of mode build (the tool builder_build_again, issue 1126 note 72): the failed key is
// built again as a new revision, and the value is written back with the revision and the asks put, or why nothing was done.
func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
res := b.BuildAgain(ctx, r.Module, r.FP, r.Why, r.By, buildAgainBound)
switch {
case res.Refused != "":
r.Answer, r.Refused = res.Refused, true
case res.Failed != "":
r.Answer, r.Revision, r.Failed = fmt.Sprintf("the key is written building as revision %d, and the build failed before any ask was put: %s", res.Revision, res.Failed), res.Revision, res.Failed
default:
r.Pending = res.Revision == 0
r.Answer, r.Revision, r.Asks = "built again: the key is written building as revision "+fmt.Sprint(res.Revision)+" and its asks are put", res.Revision, res.Asks
if res.Pending != "" {
r.Answer = res.Pending
if res.Revision != 0 {
r.Answer = "built again: the key is written building as revision " + fmt.Sprint(res.Revision) + "; " + res.Pending
}
}
}
r.Answered = time.Now().UTC()
logf("%s: build again asked by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
}
}
// announcedAgain answers a reasks value of mode announce (the tool builder_announce_again): the done key is written again
// unchanged, and the value is written back with the new revision, or why nothing was done.
func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
res := b.AnnounceAgain(ctx, r.Module, r.FP, r.Why, r.By)
if res.Refused != "" {
r.Answer, r.Refused = res.Refused, true
} else {
r.Revision, r.Answer = res.Revision, fmt.Sprintf("announced again: the done key is written again, unchanged, as revision %d", res.Revision)
}
r.Answered = time.Now().UTC()
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
}
}
// appendLog appends one entry to the builder's module log `build-log` (ADR 0297, ADR 0571 §1) under the module's key
// (delivery.BuildLogKey), on the log's own subject: a module's program appends to the logs it keeps on its own credential.
// The id makes a repeated append of one entry one entry.
func appendLog(ctx context.Context, js jetstream.JetStream, cred process.Account, e delivery.BuildLogEntry) error {
data, err := delivery.EncodeBuildLogEntry(e)
if err != nil {
return err
}
key, err := delivery.BuildLogKey(e.Module)
if err != nil {
return err
}
msg := nats.NewMsg(buscompose.LogSubject(cred.Module, delivery.BuilderLog) + "." + key)
msg.Data = data
id := e.Event + "." + e.Module + "." + e.FP + "." + e.At.Format(time.RFC3339Nano)
pub, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
return fmt.Errorf("logging %s of %s at %s: %w", e.Event, e.Module, e.FP, err)
}
return nil
}
@@ -1,73 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"log"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and
// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
// issue 774).
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
set := &buildrun.ToolchainSet{}
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
// head is told why and opens it again then.
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
src.start()
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
t.Errorf("a trunk head with the source refused: %v, want why", err)
}
if src.opened {
t.Error("a refused source counts as opened, so it is never opened again")
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
@@ -0,0 +1,211 @@
package main
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
// toolchain agreement this file repairs.
import (
"context"
"encoding/json"
"fmt"
"log"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
return false
}
if open {
g.standing[key] = since
} else {
delete(g.standing, key)
}
if !g.known {
return false
}
changed := false
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
return changed
}
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
// passed on changed.
func (g *toolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
if err != nil {
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
changed(ctx)
}
})
}()
return errs, nil
}
@@ -0,0 +1,228 @@
package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
func gateReport(t *testing.T, node, version string) (string, []byte) {
t.Helper()
key, err := delivery.ToolchainKey(node, "typescript")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
if err != nil {
t.Fatal(err)
}
return key, value
}
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
key, value := gateReport(t, node, version)
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(said) != 1 {
t.Fatalf("ace's report is said left out once; said %v", said)
}
// ace reports again while unassigned (an old value seen again): still left out.
key, value := gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
// A report deleted from the bucket goes whether its node is assigned or not.
key, _ = gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
t.Fatal("deleting g14's report passed nothing on")
}
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
// head is told why and opens it again then.
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
src.start()
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
t.Errorf("a trunk head with the source refused: %v, want why", err)
}
if src.opened {
t.Error("a refused source counts as opened, so it is never opened again")
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
}
}
@@ -9,9 +9,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/handacts"
"mesh-delivery-builder/internal/build"
)
@@ -20,27 +18,17 @@ import (
// consumer of artifacts-gone (mesh-controller internal/broker EventsStream).
const eventsStream = "EVENTS"
// SeatHeld is the seat mesh-artifact-sweep's verb held (ADR 0571 §2).
const SeatHeld = "seat:" + catalogue.ArtifactSweepSeat + ".held"
// toolCaller calls a tool or a seat verb as the builder's own account (mesh-sdk process.Conn.Call).
type toolCaller func(key string, body any) (json.RawMessage, error)
// storedVia is build.Builder's Stored: it asks the controller's verb held (mesh-sdk delivery.HeldVerb; novox/hq issue
// 1937) and answers the references the store no longer holds. One it could not ask about, with none gone, is an error:
// the key is then left done, and the controller's take-in still refuses a gone blob.
//
// It asks the seat mesh-artifact-sweep's `held` (novox/hq ADR 0571 §2), and the controller's while that seat has no holder that
// answers (a named exception of clause 10, until the lease write).
func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) {
return func(_ context.Context, references []string) ([]string, error) {
body := map[string]any{"references": references}
raw, err := call(SeatHeld, body)
if handacts.NoResponder(err) {
raw, err = call(delivery.HeldVerb, body)
}
raw, err := call(delivery.HeldVerb, map[string]any{"references": references})
if err != nil {
return nil, fmt.Errorf("asking held: %w", err)
return nil, fmt.Errorf("asking %s: %w", delivery.HeldVerb, err)
}
a, err := delivery.DecodeHeldAnswer(raw, references)
if err != nil {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4 -14
View File
@@ -1,17 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585 h1:7QoFH5qjXRpx5M4uPdYsVJ5qKl2082gq8n0lFxhoNIg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025211220-e0d1de047585/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,53 +0,0 @@
package build
import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A done build is announced again by hand** (recovery of photos and link2pay on novox, 2026-10-29). The controller takes a
// build in when a write of the build key reaches its take-in consumer, and composes then; a done key is never written again
// by the builder, so a composition or a grant lost afterwards could not be made again by any tool. AnnounceAgain writes the
// done key again with its own bytes, as a new revision of the same key: its content, state, artifacts and manifest stay
// identical, and only the write is new. Who asked and why is in the builder's log and in the reasks answer.
// AnnounceAgain rewrites the done key of module at fp unchanged, as a new revision, and answers the revision. A key that is
// not done is refused with why: failed (builder_build_again builds it) or waiting (builder_ask_again).
func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) BuildAgainResult {
b.init()
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return BuildAgainResult{Refused: err.Error()}
}
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
}
if !found {
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing to announce", key)}
}
cur, err := delivery.DecodeBuilderBuild(value)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
}
switch cur.State {
case delivery.BuildDone:
case delivery.BuildFailed:
return BuildAgainResult{Refused: fmt.Sprintf("%s is failed, not done: %s builds it again", key, BuildAgainTool)}
default:
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not done: %s asks its waiting asks again", key, cur.State, AskAgainTool)}
}
// Compare-and-set at the revision read: a build written meanwhile is not overwritten.
nrev, err := b.Claimer.Store.Update(ctx, key, value, rev)
if err != nil {
if errors.Is(err, ErrConflict) {
return BuildAgainResult{Refused: fmt.Sprintf("%s changed while it was announced: read it again", key)}
}
return BuildAgainResult{Refused: fmt.Sprintf("%s could not be written: %v", key, err)}
}
b.logf("%s: done, announced again by %s (revision %d): %s", key, by, nrev, why)
return BuildAgainResult{Revision: nrev}
}
@@ -1,49 +0,0 @@
package build
import (
"bytes"
"context"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func TestAnnounceAgainRewritesOnlyADoneKeyUnchanged(t *testing.T) { // recovery of photos and link2pay, 2026-10-29
for _, c := range []struct {
name string
key *delivery.BuilderBuild
fp string
refuse string
}{
{name: "a done key is written again, unchanged, as a new revision", key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
{name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "x"}},
{name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked"}},
{name: "a key with no build is refused", refuse: "has no build key"},
{name: "a fingerprint that is not one is refused", refuse: "fingerprint", fp: "nope"},
} {
t.Run(c.name, func(t *testing.T) {
r := newRig(t, goBundle)
var rev uint64
if c.key != nil {
c.key.Manifest = manifestD
rev = putBuild(t, r.st, r.key, *c.key)
}
before := bytes.Clone(r.st.vals[r.key])
fp := fp1
if c.fp != "" {
fp = c.fp
}
res := r.b.AnnounceAgain(context.Background(), "m", fp, "grant lost", "shanks.lead")
if c.refuse != "" {
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || r.st.revs[r.key] != rev || !bytes.Equal(r.st.vals[r.key], before) {
t.Fatalf("answer %+v, key at revision %d (was %d); want refused with %q and nothing written", res, r.st.revs[r.key], rev, c.refuse)
}
return
}
if res.Refused != "" || res.Revision <= rev || r.st.revs[r.key] != res.Revision || !bytes.Equal(r.st.vals[r.key], before) {
t.Fatalf("answer %+v, key at revision %d (was %d); want a new revision and the same bytes", res, r.st.revs[r.key], rev)
}
})
}
}
@@ -12,7 +12,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **An ask that can no longer be answered is asked again** (novox/hq issue 1945). A build seat's stream gives an ask up
@@ -21,8 +20,7 @@ import (
// restart take over the same id and wait again. The builder notices it at the check bound (CheckAfter, the step bound),
// a context's bound and no timer of its own: it asks Gone, which reads the bus, and asks such an ask again under a new
// id, written into the key, said in its log and as the event build.asked-again, at most Askings times; then the key is
// failed with why. An ask DEAD_LETTERS says the seat gave up (buildrun.LostAskWaiter, issue 734) is asked again at once.
// A person asks the same by hand with the tool builder_ask_again (AskAgain).
// failed with why. A person asks the same by hand with the tool builder_ask_again (AskAgain).
// DefaultAskings is how many times a build module's part of a build is asked, the first included, when Builder.Askings is 0.
const DefaultAskings = 3
@@ -317,11 +315,6 @@ func (b *Builder) waitOnce(ctx context.Context, ask delivery.StepAsk, again <-ch
defer stop()
}
step, err := b.Asks.Wait(wait, ask)
if err == nil && buildrun.IsLost(step) {
// The build seat gave the ask up (issue 734): it can no longer be answered, so it is asked again now, with the
// words of why, not at the check bound.
return step, step.Reason, errAgain
}
if err == nil || ctx.Err() != nil {
return step, "", err
}
@@ -1,183 +0,0 @@
package build
import (
"context"
"fmt"
"maps"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a
// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed
// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key
// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool
// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps
// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused.
// BuildAgainResult is the builder's answer to a request to build a key again.
type BuildAgainResult struct {
// Refused is why nothing was done, "" when the key was written building as a new revision.
Refused string
// Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound.
Revision uint64
Asks map[string]string
// Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal.
Pending string
// Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a
// toolchain not agreed): the new revision was written, and nothing was asked.
Failed string
}
// handWatch follows one request to build a key again until its asks are put.
type handWatch struct {
rev uint64
lanes int
asks map[string]string
done chan struct{}
shut bool
// ended is set when the look ended before the asks were all put.
ended bool
}
// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
func (w *handWatch) settle() {
if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes {
w.shut = true
close(w.done)
}
}
func (b *Builder) hand(key string, f func(w *handWatch)) {
b.mu.Lock()
defer b.mu.Unlock()
if w := b.hands[key]; w != nil {
f(w)
}
}
func (b *Builder) handClaimed(key string, rev uint64) {
b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() })
}
func (b *Builder) handLanes(key string, n int) {
b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() })
}
func (b *Builder) handAsk(key, language, id string) {
b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() })
}
// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
func (b *Builder) handEnded(key string) {
b.hand(key, func(w *handWatch) {
if !w.shut {
w.shut, w.ended = true, true
close(w.done)
}
})
}
// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
// ctx is the builder's own context: the build it starts lives on it.
func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
b.init()
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return BuildAgainResult{Refused: err.Error()}
}
value, _, found, err := b.Claimer.Store.Get(ctx, key)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
}
if !found {
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)}
}
cur, err := delivery.DecodeBuilderBuild(value)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
}
switch {
case cur.State == delivery.BuildDone:
return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued:
return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
case cur.State != delivery.BuildFailed:
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)}
case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix):
return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)}
}
// One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint,
// and the watch and the look are registered before anything else can take the key.
w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})}
b.mu.Lock()
m, known := b.known[module]
switch {
case !known || m.FP2 != fp:
b.mu.Unlock()
return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)}
case m.Manifest == "":
b.mu.Unlock()
return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)}
case b.inflight[key] || b.hands[key] != nil:
b.mu.Unlock()
return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)}
}
b.hands[key] = w
b.inflight[key] = true
b.mu.Unlock()
defer func() {
b.mu.Lock()
if b.hands[key] == w {
delete(b.hands, key)
}
b.mu.Unlock()
}()
b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why)
b.startOnce(ctx, key, module, m)
wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8)
defer stop()
select {
case <-w.done:
case <-wait.Done():
}
b.mu.Lock()
res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
ended, shut, lanes := w.ended, w.shut, w.lanes
b.mu.Unlock()
switch {
case res.Revision == 0 && shut:
res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
case res.Revision == 0:
res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
// The look ended before every ask was put: say how the key stands, not that it was asked.
if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found {
if cur, err := delivery.DecodeBuilderBuild(v); err == nil {
switch cur.State {
case delivery.BuildFailed:
res.Failed = cur.Error
case delivery.BuildDone:
default:
res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step)
}
}
}
case !shut:
res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound)
}
return res
}
func boolInt(b bool) int {
if b {
return 1
}
return 0
}
@@ -68,7 +68,7 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
continue
}
m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path,
Toolchains: b.agreedFor(ctx, cm.Manifest), Packages: cm.Packages} // the group's package heads (ADR 0573 §4)
Toolchains: b.agreedFor(ctx, cm.Manifest)}
if len(cm.Paths) > 0 {
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
@@ -7,7 +7,6 @@ import (
"fmt"
"maps"
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -69,11 +68,6 @@ type Builder struct {
Askings int
// Said is told each ask asked again (the builder's event build.asked-again). Nil tells nobody.
Said func(AskedAgain)
// Log is told each build's start and end, the entries of the builder's log (novox/hq ADR 0571 §1). Nil tells nobody.
Log func(delivery.BuildLogEntry)
// Now is the clock of the log; nil is time.Now. Node is the node the builder runs on, which asked the build.
Now func() time.Time
Node string
once sync.Once
sem chan struct{}
@@ -95,8 +89,6 @@ type Builder struct {
held map[string]bool
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
waiting map[string]map[string]chan string
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
hands map[string]*handWatch
}
type planned struct {
@@ -114,7 +106,6 @@ func (b *Builder) init() {
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
b.hands = map[string]*handWatch{}
})
}
@@ -381,35 +372,9 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module
}()
}
// startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and
// a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does.
func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery.Module) {
b.wg.Add(1)
go func() {
defer b.wg.Done()
b.sem <- struct{}{}
defer func() { <-b.sem }()
for {
b.try(ctx, module, m)
b.mu.Lock()
next, more := b.again[key]
delete(b.again, key)
if !more || ctx.Err() != nil {
delete(b.inflight, key)
b.mu.Unlock()
return
}
b.mu.Unlock()
m = next // a look the builder queued meanwhile is not lost (consider's rule); it is not a second by-hand build
}
}()
}
// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
key, _ := delivery.BuilderBuildKey(module, m.FP2)
defer b.handEnded(key)
plan, planErr, read := b.planOf(ctx, m.Manifest)
nothing := read && planErr == nil && len(plan.Tasks) == 0
if planErr == nil && !nothing {
@@ -420,6 +385,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
return
}
}
key, _ := delivery.BuilderBuildKey(module, m.FP2)
b.mu.Lock()
trunk := b.trunkFP[key]
first := !b.tried[key]
@@ -434,8 +400,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
if cl == nil {
return
}
b.handClaimed(key, cl.Rev)
b.logBuild(LogStarted(module, m, b.refOf(module, m.FP2), b.Node, b.now()))
if cl.Unasked != nil {
b.logf("%v", cl.Unasked)
}
@@ -485,7 +449,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
b.logf("%s: the build could not be written: %v", cl.Key, err)
return
}
b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result))
if result.State == delivery.BuildFailed {
b.logf("%s: failed: %s", cl.Key, result.Error)
}
@@ -572,9 +535,6 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
}
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
b.handLanes(key, len(lanes))
}
var wg sync.WaitGroup
for _, l := range lanes {
wg.Add(1)
@@ -757,11 +717,7 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
v.Asks = map[string]string{}
}
v.Asks[language] = id
if err := n.b.Claimer.Finish(ctx, n.cl, v); err != nil {
return err
}
n.b.handAsk(n.cl.Key, language, id)
return nil
return n.b.Claimer.Finish(ctx, n.cl, v)
}
// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the
@@ -808,11 +764,6 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
job.Fetch = req.Args()
}
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: language, Asker: delivery.BuilderModule, At: time.Now(), Build: job}
if language == delivery.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
// them into the workspace, whatever the repository's go.mod names.
ask.Packages = maps.Clone(m.Packages)
}
if id := asked.resume(language); id != "" {
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
if b.Held != nil {
@@ -885,60 +836,3 @@ func needsFetch(tasks []Task) bool {
}
return false
}
func (b *Builder) now() time.Time {
if b.Now != nil {
return b.Now()
}
return time.Now()
}
// logBuild tells the log an entry, said in the journal when it cannot be one.
func (b *Builder) logBuild(e delivery.BuildLogEntry) {
if b.Log == nil {
return
}
if _, err := delivery.EncodeBuildLogEntry(e); err != nil {
b.logf("%s at %s: no log entry: %v", e.Module, e.FP, err)
return
}
b.Log(e)
}
// refOf is the ref a candidate's build of module's fingerprint is built for, "" for the trunk's.
func (b *Builder) refOf(module, fp string) string {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return ""
}
b.mu.Lock()
defer b.mu.Unlock()
return b.via[key].Ref
}
// LogStarted is the log's entry of a build of module's fingerprint starting (ADR 0571 §1): the module, the fingerprint, the
// source revision, where it is built from (repository, directory, commit, the candidate's ref), the node of the builder
// that asked it and the time.
func LogStarted(module string, m delivery.Module, ref, by string, at time.Time) delivery.BuildLogEntry {
dir := m.Path
if dir == "" {
dir = delivery.RootPath
}
return delivery.BuildLogEntry{Event: delivery.BuildStartedEvent, Module: module, FP: m.FP2, Source: strconv.FormatUint(m.Src, 10),
Repository: m.Repository, Directory: dir, Commit: m.Commit, Ref: ref, Node: by, At: at.UTC()}
}
// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it
// was built against and each artifact made, as the done key has them: what mesh-catalog registers a build from.
func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry {
e := LogStarted(module, m, ref, by, at)
e.Event, e.Outcome, e.Manifest = delivery.BuildEndedEvent, result.State, m.Manifest
if result.Manifest != "" {
e.Manifest = result.Manifest
}
if m.Inputs2 != nil {
e.Against = append([]string(nil), m.Inputs2.Bases...)
}
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
return e
}
@@ -5,7 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"reflect"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strconv"
"strings"
"sync"
@@ -13,11 +13,9 @@ import (
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the
@@ -223,42 +221,7 @@ var (
func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
r := newRig(t, goBundle, archiveA, tsBundle)
var logged []delivery.BuildLogEntry
var lmu sync.Mutex
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) { lmu.Lock(); logged = append(logged, e); lmu.Unlock() }
v := r.consider(t, "modules/m")
// 1955/L7 (ADR 0571 §1): the build is one start and one end in the builder's log, every field named, its duration read
// from the two.
if len(logged) != 2 || logged[0].Event != delivery.BuildStartedEvent || logged[1].Event != delivery.BuildEndedEvent ||
logged[1].Outcome != delivery.BuildDone || len(logged[1].Made) != 3 || logged[1].Made[0].Reference == "" ||
logged[1].Manifest != manifestD || logged[0].Manifest != "" || logged[0].Repository != "novox/mesh-catalog" ||
logged[0].Directory != "modules/m" || logged[0].Commit != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || logged[0].Node != "novox" ||
logged[0].FP != fp1 || logged[0].Module != "m" {
t.Fatalf("logged = %+v", logged)
}
for _, e := range logged {
if _, err := delivery.EncodeBuildLogEntry(e); err != nil {
t.Errorf("an entry the log refuses: %v", err)
}
}
if _, ok := delivery.BuildDuration(logged[0], logged[1]); !ok {
t.Error("no duration from the two entries")
}
// builder_builds_since (ADR 0571 clause 9) answers the end alone from the log's page, and says when the log holds more.
var page stdio.LogPage
for i, e := range logged {
raw, _ := delivery.EncodeBuildLogEntry(e)
page.Entries = append(page.Entries, stdio.LogEntry{Sequence: uint64(i + 1), Key: e.Module, Value: raw, At: e.At})
}
page.Next = 3
since, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At)
if err != nil || len(since.Builds) != 1 || since.Builds[0].Event != delivery.BuildEndedEvent || len(since.Builds[0].Made) != 3 || !since.More {
t.Errorf("builds since = %+v, %v, want the one end and more", since, err)
}
page.Entries[0].Value = []byte(`{"v":0}`)
if _, err := BuildsSince(func(time.Time, string, int) (*stdio.LogPage, error) { return &page, nil }, logged[0].At); err == nil {
t.Error("an entry that does not read was skipped, want it refused")
}
if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 {
t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v)
}
@@ -301,14 +264,7 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
r := newRig(t, goBundle)
r.asks.Answer = nil // no node has pulled it yet
// Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with.
sdk := delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')}
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
FP2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: agreedToolchains, Packages: sdk})
if err != nil {
t.Fatal(err)
}
r.b.Module(context.Background(), "m", data)
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
var asks []delivery.StepAsk
for deadline := time.Now().Add(2 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
@@ -320,9 +276,6 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
if len(asks) != 1 || a.Kind != delivery.AskDelivery || a.Language != delivery.AskGo || a.Build == nil || a.Build.FP != fp1 {
t.Fatalf("asks = %+v, want one Go delivery ask of the fingerprint", asks)
}
if !reflect.DeepEqual(a.Packages, sdk) {
t.Errorf("the ask's packages = %v, want %v", a.Packages, sdk)
}
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding || v.Asks[delivery.AskGo] != a.ID {
t.Fatalf("the key = %+v, want building naming the ask %s", v, a.ID)
}
@@ -737,130 +690,17 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
step string
byHand bool
always bool // Gone says every ask cannot be answered, not only the first
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
deaf bool // the ask's own watch hears nothing though its step ends (the lost check of 2026-10-28)
puts int
done bool
asking int // the asking the event says, 0 when none is said
// again, when set, is a request to build the key again by hand (BuildAgain, issue 1126 note 72) of the key as
// it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it
// is refused, "" when it is built again as a new revision and its ask id answered.
again bool
key *delivery.BuilderBuild
known string
refuse string
// edit changes the module the builder has read, busy marks the key looked at (1) or asked by hand (2), failed is a part of
// the error the key then holds when its build ends before its asks are put.
edit func(m *delivery.Module)
busy int
failed string
// during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards.
during bool
}{
{name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true,
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a key looked at by the builder now is refused, not built twice", again: true, refuse: "is being looked at", busy: 1,
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a key already asked by hand is refused, its watch kept", again: true, refuse: "is being looked at", busy: 2,
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *delivery.Module) { m.Manifest = "" },
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing",
edit: func(m *delivery.Module) {
m.Toolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}
},
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a build that fails before its asks says the key failed, not that it was asked", again: true, failed: "cannot be read",
edit: func(m *delivery.Module) { m.Manifest = mdigest('b') },
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a failed key is written building as a new revision and asks the seat", again: true,
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}},
{name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do",
key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
{name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting",
key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}},
{name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry",
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}},
{name: "build again: a key with no build is refused", again: true, refuse: "has no build key"},
{name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64),
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
{name: "its step ended while its watch was deaf: read at its build module's next step end, done with no new ask", deaf: true, puts: 1, done: true},
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
} {
t.Run(c.name, func(t *testing.T) { // 1945
r := newRig(t, goBundle)
if c.again {
var rev uint64
if c.key != nil {
c.key.Manifest = manifestD
rev = putBuild(t, r.st, r.key, *c.key)
}
m, err := delivery.DecodeModule(moduleValue(t, "modules/m"))
if err != nil {
t.Fatal(err)
}
if c.known != "" {
m.FP2 = c.known
}
if c.edit != nil {
c.edit(&m)
}
r.b.init()
var stored atomic.Int64
if c.during {
r.asks.Answer = nil
r.b.Stored = func(context.Context, []string) ([]string, error) { stored.Add(1); return nil, nil }
}
switch c.busy {
case 1:
r.b.inflight[r.key] = true
case 2:
r.b.hands[r.key] = &handWatch{}
}
r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked
log := &logged{}
r.b.Logf = log.logf
res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second)
if c.during {
// The trunk's modules value arrives while the by-hand build waits for its ask: it is queued, and run after it.
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || res.Revision <= rev || stored.Load() != 1 {
t.Fatalf("key %+v, answer %+v, %d asks of the store; want built from the new revision and the queued look run once after it", v, res, stored.Load())
}
return
}
r.b.Wait()
after := r.st.revs[r.key]
if c.refuse != "" {
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || len(r.asks.Asks()) != 0 || after != rev {
t.Fatalf("refused %q (revision %d, %d asks, key at revision %d was %d); want refused with %q and nothing written or asked", res.Refused, res.Revision, len(r.asks.Asks()), after, rev, c.refuse)
}
return
}
if c.failed != "" {
if !strings.Contains(res.Failed, c.failed) || res.Revision <= rev || len(res.Asks) != 0 || len(r.asks.Asks()) != 0 {
t.Fatalf("answer %+v after %d asks; want a new revision (was %d), no ask and the key's failure %q", res, len(r.asks.Asks()), rev, c.failed)
}
return
}
asks := r.asks.Asks()
if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID {
t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev)
}
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision {
t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision)
}
if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") {
t.Errorf("the why is not logged: %v", log.lines)
}
return
}
r.asks.Answer = nil
r.b.Askings = c.askings
var said []AskedAgain
@@ -874,16 +714,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
r.st.vals[r.key] = data
}
gone := &goneAsks{}
dead := &buildrun.DeadLetters{}
if c.dead {
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
}
var heard *buildrun.HeardEnds
if c.deaf {
heard = &buildrun.HeardEnds{Inner: silentWatch{}, Read: r.asks}
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second}
}
if !c.byHand && !c.dead && !c.deaf {
if !c.byHand {
gone.why = func(a delivery.StepAsk) string {
if c.always || a.ID == first || first == "" {
return "it waits first in line while a node took a later ask"
@@ -893,10 +724,6 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond
}
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
if c.dead {
dead.Heard(buildrun.GivenUp{AskID: waitForAsks(t, r, 1)[0].ID, Stream: "SEAT_NODE_BUILD_GO", Deliveries: 5,
At: time.Date(2026, 10, 26, 12, 26, 40, 0, time.UTC)})
}
if c.byHand {
waitForAsks(t, r, 1)
for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) {
@@ -905,16 +732,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
}
}
}
if c.deaf {
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
for deadline := time.Now().Add(2 * time.Second); heard.Waiting() == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("no ask is waited on")
}
}
heard.Nudge(buildrun.GoBuildModule) // a step of mesh-build-go ended
}
if c.done && !c.deaf {
if c.done {
asks := waitForAsks(t, r, c.puts)
if first == "" && !c.byHand {
first = asks[0].ID
@@ -939,16 +757,13 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
mu.Lock()
defer mu.Unlock()
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && !c.deaf && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
}
gone.mu.Unlock()
if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") {
t.Errorf("said %+v, want once at asking %d", said, c.asking)
}
if c.dead && (len(said) != 1 || !strings.Contains(said[0].Why, buildrun.LostWords+" after 5 deliveries at 2026-10-26T12:26:40Z")) {
t.Errorf("said %+v, want the ask asked again saying the seat gave it up", said)
}
})
}
t.Run("what the bus says of an ask", func(t *testing.T) { // 1945
@@ -1021,17 +836,12 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at})
_, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at)
r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}),
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
Hearing: state(map[string]string{buildrun.HearingKey: `{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],"why":"DEAD_LETTERS is not heard: SEAT_NODE_BUILD_GO: permissions violation","at":"2026-10-26T13:00:00Z"}`}),
Now: func() time.Time { return at.Add(time.Hour) }}
Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})},
Now: func() time.Time { return at.Add(time.Hour) }}
out, err := r.Stuck("")
if err != nil {
t.Fatal(err)
}
// Issue 734, review round 1: a builder that does not hear DEAD_LETTERS says so, with why.
if dl := out["dead_letters"].(map[string]any); dl["heard"] != false || !strings.Contains(dl["why"].(string), "permissions violation") {
t.Errorf("dead_letters = %v, want not heard, saying why", dl)
}
stuck := out["stuck"].([]Stuck)
if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" {
t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck)
@@ -1149,10 +959,3 @@ func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) {
t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks()))
}
}
// silentWatch is a watch of an ask's step that stays open and hears nothing.
type silentWatch struct{}
func (silentWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
return make(chan delivery.Step), nil
}
@@ -1,43 +0,0 @@
package build
import (
"fmt"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// BuildsSinceLimit is how many entries one answer of BuildsSinceTool reads of the log; More says the log held more.
const BuildsSinceLimit = 500
// LogSince is one page of the builder's log from a time on: stdio.Log(BuilderLog).Since.
type LogSince func(at time.Time, keys string, limit int) (*stdio.LogPage, error)
// BuildsSince answers delivery.BuildsSinceTool (ADR 0571 clause 9): every ended build in the builder's log from since on,
// oldest first, read through the runtime's log verbs; More when the page was not the log's last, so the caller asks again
// from the last build's time. An entry the log holds that does not read is refused, naming its sequence: a reader that
// skipped it would register too little.
func BuildsSince(read LogSince, since time.Time) (delivery.BuildsSince, error) {
out := delivery.BuildsSince{Since: since.UTC(), Builds: []delivery.BuildLogEntry{}}
page, err := read(since, "", BuildsSinceLimit)
if err != nil {
return out, fmt.Errorf("the builder's log %s from %s: %w", delivery.BuilderLog, since.Format(time.RFC3339), err)
}
if page == nil {
return out, nil
}
var entries []delivery.BuildLogEntry
for _, le := range page.Entries {
e, err := delivery.DecodeBuildLogEntry(le.Value)
if err != nil {
return out, fmt.Errorf("the builder's log %s, entry %d: %w", delivery.BuilderLog, le.Sequence, err)
}
entries = append(entries, e)
}
if ended := delivery.EndedSince(entries, since); ended != nil {
out.Builds = ended
}
out.More = page.Next != 0
return out, nil
}
@@ -22,11 +22,6 @@ import (
const (
StuckTool = "builder_stuck"
AskAgainTool = "builder_ask_again"
// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
BuildAgainTool = "builder_build_again"
// AnnounceAgainTool writes a done key again unchanged, so the controller takes it in and composes again: a reasks value
// of Mode ModeAnnounce.
AnnounceAgainTool = "builder_announce_again"
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
ReAsksState = "reasks"
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
@@ -41,27 +36,8 @@ type ReAsk struct {
At time.Time `json:"at"`
Answer string `json:"answer,omitempty"`
Answered time.Time `json:"answered,omitzero"`
// Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again.
Mode string `json:"mode,omitempty"`
Module string `json:"module,omitempty"`
FP string `json:"fp,omitempty"`
// Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new
// revision (building) and the ask ids it put by language.
Refused bool `json:"refused,omitempty"`
// Pending says the builder answered before the key was claimed or all its asks were put: neither a refusal nor done.
// Failed is the error of a key written building that failed before its asks were put.
Pending bool `json:"pending,omitempty"`
Failed string `json:"failed,omitempty"`
Revision uint64 `json:"revision,omitempty"`
Asks map[string]string `json:"asks,omitempty"`
}
// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
const ModeBuild = "build"
// ModeAnnounce is the Mode of a reasks value that asks a done key to be announced again.
const ModeAnnounce = "announce"
// DecodeReAsk reads a reasks value.
func DecodeReAsk(data []byte) (ReAsk, error) {
var r ReAsk
@@ -87,32 +63,6 @@ func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error)
return key, data, err
}
// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value.
func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return "", nil, err
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why the key is built again (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp})
return key, data, err
}
// NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value.
func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return "", nil, err
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why the done build is announced again (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeAnnounce, Module: module, FP: fp})
return key, data, err
}
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
type StateRead struct {
Keys func() ([]string, error)
@@ -156,9 +106,7 @@ type StuckReader struct {
ReAsks StateRead
// Steps are the build modules' steps states, by build module.
Steps map[string]StateRead
// Hearing is the builder's state hearing: whether it hears DEAD_LETTERS (issue 734); without it none is said.
Hearing StateRead
Now func() time.Time
Now func() time.Time
// Logf, when set, gets the one line a call logs: the values it read and the time it took.
Logf func(format string, args ...any)
}
@@ -201,20 +149,6 @@ func parallel(n int, read func(i int) error) error {
}
// Stuck is every key building or queued with its asks, oldest first; with module, that module's only.
// deadLetters says whether the builder hears the asks the build seats give up on (issue 734), and why not: deaf, a lane
// whose ask is given up waits for its check bound (buildrun.HearingHealth over the kept hearing).
func (r StuckReader) deadLetters() map[string]any {
if r.Hearing.Get == nil {
return map[string]any{"heard": false, "why": "the builder's state hearing is not readable here"}
}
raw, _, err := r.Hearing.Get(buildrun.HearingKey)
if err != nil {
return map[string]any{"heard": false, "why": "the builder's state hearing cannot be read: " + err.Error()}
}
a := buildrun.HearingHealth(raw)
return map[string]any{"heard": a.Healthy, "why": a.Why}
}
func (r StuckReader) Stuck(module string) (map[string]any, error) {
started := time.Now()
reads := 0
@@ -334,7 +268,6 @@ func (r StuckReader) Stuck(module string) (map[string]any, error) {
})
answer := map[string]any{"stuck": out, "count": len(out),
"delivery_counts": "not readable by the builder: the build seats' workers' information is granted to the delivery view alone (its backlog); an ask given up on is told by the builder at its check bound and asked again"}
answer["dead_letters"] = r.deadLetters()
if cut {
answer["cut"], answer["note"] = true, fmt.Sprintf("read %d of %d build keys in key order and stopped; give a module to read its keys alone", MaxReads, total)
}
+11 -23
View File
@@ -10,8 +10,6 @@ import (
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
@@ -42,14 +40,10 @@ type manifest struct {
Emits []string `json:"emits"`
Consumes []string `json:"consumes"`
State []state `json:"state"`
Logs []struct {
Name string `json:"name"`
MaxAgeDays int `json:"max-age-days"`
} `json:"logs"`
Reads []string `json:"reads"`
Uses []string `json:"uses"`
Invokes []string `json:"invokes"`
Config map[string]struct {
Reads []string `json:"reads"`
Uses []string `json:"uses"`
Invokes []string `json:"invokes"`
Config map[string]struct {
Type string `json:"type"`
Description string `json:"description"`
Pattern string `json:"pattern"`
@@ -93,13 +87,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
if m.Module != "mesh-delivery-builder" {
t.Errorf("module = %q", m.Module)
}
if want := []state{{Name: "builds", History: 5}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
t.Errorf("state = %+v, want %+v", m.State, want)
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains",
"mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
@@ -107,14 +101,12 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
}
// 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no
// other tool; it holds no capacity.
// 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's only while the seat has no holder (clause 10).
if want := []string{"seat:mesh-artifact-sweep.held", "seat:mesh-controller.held"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 {
if want := []string{"seat:mesh-controller.held"}; !reflect.DeepEqual(m.Invokes, want) || len(m.Config) != 0 {
t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want)
}
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it
// starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4).
// 1955/L7: and the seat mesh-artifact-sweep's, the controller's until the lease write (its take-in sweeps until then).
if want := []string{"mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
@@ -124,17 +116,13 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
t.Errorf("emits = %v, want %v", m.Emits, want)
}
// 1955/L7 (ADR 0571 §1): its module log, each build's start and end, kept 30 days; no bare stream.
if len(m.Logs) != 1 || m.Logs[0].Name != delivery.BuilderLog || m.Logs[0].MaxAgeDays != delivery.BuilderLogDays {
t.Errorf("logs = %+v, want %s kept %d days", m.Logs, delivery.BuilderLog, delivery.BuilderLogDays)
}
for _, k := range []string{"streams", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the builder has none", k)
}
+2 -10
View File
@@ -3,10 +3,7 @@
"version": "1",
"slug": "builder",
"tools": [
"builder_announce_again",
"builder_ask_again",
"builder_build_again",
"builder_builds_since",
"builder_seat_queue",
"builder_stuck"
],
@@ -14,32 +11,26 @@
"build.asked-again"
],
"consumes": [
"mesh-artifact-sweep.artifacts-gone",
"mesh-build-go.toolchain.reported",
"mesh-build-node.toolchain.reported",
"mesh-controller.artifacts-gone"
],
"invokes": [
"seat:mesh-artifact-sweep.held",
"seat:mesh-controller.held"
],
"state": [
{ "name": "builds", "history": 5 },
{ "name": "hearing", "history": 1 },
{ "name": "instances", "history": 1 },
{ "name": "queues", "history": 1 },
{ "name": "reasks", "history": 1 }
],
"logs": [
{ "name": "build-log", "max-age-days": 30 }
],
"reads": [
"mesh-build-go.steps",
"mesh-build-go.toolchains",
"mesh-build-image.steps",
"mesh-build-node.steps",
"mesh-build-node.toolchains",
"mesh-delivery-assigner.placement",
"mesh-controller.assignments",
"mesh-delivery-gate.source",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
@@ -50,6 +41,7 @@
"node-build-image",
"node-build-node"
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
@@ -1,6 +1,6 @@
// Command mesh-delivery-checker-tools serves the checker's read tools (novox/hq ADR 0484 clause 13, issue 1686): the runtime
// launches it for a call, it reads the checker's verdicts and kept states and the indexer's candidates through the checker's
// state and reads. It writes only the tries (the try's tools) and, asking a check again by hand, one verdict. The checker is the other program: it serves no tool (ADR 0437 §4).
// state and reads, and writes nothing. The checker is the other program: it serves no tool (ADR 0437 §4).
package main
import (
@@ -12,7 +12,6 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-checker/internal/check"
)
@@ -54,36 +53,10 @@ func read(s stdio.KeptState) check.StateRead {
}}
}
// runsOf says which commit the mesh runs of a module from the indexer's modules state, as the checker's own record of it does
// (the commit the module's record was computed at); "" when the record is not readable.
type runsOf struct{ modules check.StateRead }
func (r runsOf) CommitOf(module string) string {
if r.modules.Get == nil {
return ""
}
raw, err := r.modules.Get(module)
if err != nil || len(raw) == 0 {
return ""
}
m, err := delivery.DecodeModule(raw)
if err != nil {
return ""
}
return m.Commit
}
// stateStore is a state the tools write: the tries.
type stateStore struct {
check.StateRead
create func(key string, value []byte) error
entry func(key string) (json.RawMessage, uint64, error)
update func(key string, value []byte, revision uint64) error
}
func (s stateStore) Entry(key string) (json.RawMessage, uint64, error) { return s.entry(key) }
func (s stateStore) Update(key string, value []byte, revision uint64) error {
return s.update(key, value, revision)
}
func (s stateStore) Keys() ([]string, error) { return s.StateRead.Keys() }
@@ -93,21 +66,19 @@ func (s stateStore) Create(key string, value []byte) error { return s.create(k
// Trying is what the try's tools work on: the tries, and the gate's trunk heads.
type Trying struct {
Tries check.TryStore
Ender check.TryEnder
Trunks func() (map[string]string, error)
Now func() time.Time
}
// TryTools are the try's tools (ADR 0501 §5): ask one, read one, and end one by hand (issue 734). mesh-controller's try calls them.
// TryTools are the try's two tools (ADR 0501 §5): ask one, read one. mesh-controller's try calls them.
func TryTools(tr Trying) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
return []stdio.Tool{
{Name: check.TryTool,
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
@@ -135,8 +106,7 @@ func TryTools(tr Trying) []stdio.Tool {
}},
{Name: check.TryReadTool,
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
"the head and the trunk commit it was merged onto, the time it took and what failed first. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
Run: func(args map[string]any) (any, error) {
@@ -145,46 +115,9 @@ func TryTools(tr Trying) []stdio.Tool {
}
return check.ReadTry(tr.Tries, str(args, "id"))
}},
{Name: check.TryEndTool,
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
"takes it within the step bound. A try that ended already is refused.",
Input: map[string]any{"type": "object", "required": []string{"id", "why"}, "properties": map[string]any{
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
"why": map[string]any{"type": "string", "description": "why it is ended, in words"}}},
Run: func(args map[string]any) (any, error) {
if tr.Ender == nil {
return nil, fmt.Errorf("the tries state cannot be written here")
}
now := time.Now
if tr.Now != nil {
now = tr.Now
}
t, err := check.EndTry(tr.Ender, str(args, "id"), str(args, "why"), now())
if err != nil {
return nil, err
}
return map[string]any{"id": t.ID, "state": t.State, "said": t.Said}, nil
}},
}
}
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
const Seat = "mesh-delivery-checker"
// SeatTryTools are the try's tools as the seat's verbs: try, try-read and try-end, run by the same code as checker_try,
// checker_try_read and checker_try_end, so the answer is the same. The module tools stay beside them.
func SeatTryTools(tr Trying) []stdio.Tool {
verbs := map[string]string{check.TryTool: "try", check.TryReadTool: "try-read", check.TryEndTool: "try-end"}
var out []stdio.Tool
for _, t := range TryTools(tr) {
t.Name = Seat + "." + verbs[t.Name]
out = append(out, t)
}
return out
}
// Tools are the read tools over the checker's states.
func Tools(r check.Reader) []stdio.Tool {
return []stdio.Tool{
@@ -241,33 +174,12 @@ func Tools(r check.Reader) []stdio.Tool {
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return r.Status() }},
}
}
// RecoveryTools are the tools that steer a check by hand (tools-are-how-a-module-is-spoken-to): ask a check again whose
// wait judged nothing (the lost check of 2026-10-28). They write the checker's verdicts through w.
func RecoveryTools(r check.Reader, w check.VerdictEntry) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
return []stdio.Tool{{Name: check.AskAgainTool,
Description: "Ask a candidate's check again by hand, whose verdict stays running while nothing runs for it (" + check.StatusReadTool + " names it " +
"under lost): its verdict is ended as asked again, saying why, and the checker, which hears its verdicts, stops the wait that judged nothing " +
"and asks the check anew, whole. Refused for a group with no candidate, a verdict that is not running (a pass or failure stands until a push or " +
"a trunk move), and while a step of its pull requests can still run on a build module (its end judges it). The checker reads a step that " +
"ended while its watch was deaf by itself at the next step end of its build module; this is for what that does not reach.",
Input: map[string]any{"type": "object", "required": []string{"group", "why"}, "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in words; kept in the verdict it ends"}}},
Run: func(args map[string]any) (any, error) {
return r.AskAgain(w, str(args, "group"), str(args, "why"))
}}}
}
// stepBound is the checker's step bound as the checker reads it (MESH_CHECKER_STEP_BOUND), its default when unset or not a
// duration.
func stepBound() time.Duration {
@@ -281,10 +193,7 @@ func main() {
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
PreviewState: read(stdio.State("previews")),
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)),
AskState: read(stdio.State(check.AsksState))}
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
tries := stdio.State(check.TriesState)
source := stdio.State("mesh-delivery-gate.source")
trunks := func() (map[string]string, error) {
@@ -296,35 +205,14 @@ func main() {
return s.Heads, err
}
r.Trunks = trunks
store := stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
_, err := tries.Create(key, json.RawMessage(value))
return err
}, entry: func(key string) (json.RawMessage, uint64, error) {
reads.Add(1)
e, err := tries.Get(key)
if err != nil || e == nil {
return nil, 0, err
}
return e.Value, e.Revision, nil
}, update: func(key string, value []byte, revision uint64) error {
_, err := tries.Update(key, json.RawMessage(value), revision)
return err
}}
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
verdicts := stdio.State("verdicts")
written := stateStore{entry: func(key string) (json.RawMessage, uint64, error) {
reads.Add(1)
e, err := verdicts.Get(key)
if err != nil || e == nil {
return nil, 0, err
}
return e.Value, e.Revision, nil
}, update: func(key string, value []byte, revision uint64) error {
_, err := verdicts.Update(key, json.RawMessage(value), revision)
return err
}}
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
if err := stdio.Serve("", logged(tools)); err != nil {
trying := Trying{
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
_, err := tries.Create(key, json.RawMessage(value))
return err
}},
Trunks: trunks,
}
if err := stdio.Serve("", logged(append(Tools(r), TryTools(trying)...))); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
}
@@ -23,14 +23,6 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
if !slices.Equal(names, want) {
t.Errorf("tools = %v, want %v", names, want)
}
// The recovery tool beside them asks a lost check again; with nothing to write it refuses.
recovery := RecoveryTools(check.Reader{}, nil)
if len(recovery) != 1 || recovery[0].Name != check.AskAgainTool || recovery[0].Description == "" {
t.Errorf("recovery tools = %+v", recovery)
}
if _, err := recovery[0].Run(map[string]any{"group": "feat/x", "why": "lost"}); err == nil {
t.Errorf("%s answered with no state to write", check.AskAgainTool)
}
// With nothing to read from a tool says so and changes nothing.
for _, tl := range Tools(check.Reader{}) {
if tl.Name == check.StatusReadTool {
@@ -61,19 +53,13 @@ func (m *memStore) Create(key string, value []byte) error {
return nil
}
func (m *memStore) Entry(key string) (json.RawMessage, uint64, error) { return m.vals[key], 1, nil }
func (m *memStore) Update(key string, value []byte, revision uint64) error {
m.vals[key] = value
return nil
}
func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed interface with mesh-controller's try
store := &memStore{vals: map[string]json.RawMessage{}}
tools := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
tools := TryTools(Trying{Tries: store, Now: func() time.Time { return time.Unix(1, 5) },
Trunks: func() (map[string]string, error) {
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
}})
if len(tools) != 3 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" || tools[2].Name != "checker_try_end" {
if len(tools) != 2 || tools[0].Name != "checker_try" || tools[1].Name != "checker_try_read" {
t.Fatalf("tools = %+v", tools)
}
out, err := tools[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
@@ -88,32 +74,6 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
if err != nil || !strings.Contains(string(raw.(json.RawMessage)), `"state":"queued"`) {
t.Errorf("read = %s, %v", raw, err)
}
// The seat's verbs answer as the tools do (ADR 0547 §10, issue 774): the same ask is the same try, and the read is alike.
verbs := SeatTryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(1, 5) },
Trunks: func() (map[string]string, error) {
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
}})
if len(verbs) != 3 || verbs[0].Name != Seat+".try" || verbs[1].Name != Seat+".try-read" || verbs[2].Name != Seat+".try-end" {
t.Fatalf("verbs = %+v", verbs)
}
again, err := verbs[0].Run(map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40), "trunk": "main"})
if err != nil || again.(map[string]any)["id"] != id {
t.Errorf("verb try = %v, %v, want the waiting try %s", again, err, id)
}
if viaVerb, err := verbs[1].Run(map[string]any{"id": id}); err != nil || string(viaVerb.(json.RawMessage)) != string(raw.(json.RawMessage)) {
t.Errorf("verb try-read = %s, %v; tool = %s", viaVerb, err, raw)
}
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
t.Error("the verb tried a repository nothing checks")
}
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
if err != nil || ended.(map[string]any)["state"] != "error" {
t.Errorf("end = %v, %v", ended, err)
}
if _, err := tools[2].Run(map[string]any{"id": id, "why": "again"}); err == nil {
t.Error("an ended try was ended again")
}
if _, err := tools[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil ||
err.Error() != "no level of novox/hq is checked by the checker" {
t.Errorf("hq: %v", err)
@@ -9,7 +9,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
)
@@ -125,61 +124,9 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
return parseManifest(kept.JSON)
}
// parseManifest decodes a manifest the indexer kept, as every reader does (novox/hq ADR 0578 §3): judged by nothing here,
// a field it does not know set aside. The checker judges a change's manifests with the indexer's judge, run as a step
// (judge.go).
// parseManifest reads a manifest the indexer kept as the indexer and the assigner read it (issues 1751, 1906): a config value that
// falls back to a predefined key is judged by the merge gate in the catalogue, against the predefined.json at its root that this
// process never holds, so it is not unreadable here (novox/hq ADR 0348 §7).
func parseManifest(raw string) (catalogue.Manifest, error) {
return catalogue.Decode([]byte(raw))
}
// awaitedKV is a bucket that may appear after the start (process.AwaitBucket) as a check.KV: a call while it is not on the
// bus fails saying so.
type awaitedKV struct{ b *process.AwaitedBucket }
func (a awaitedKV) kv() (kvStore, error) {
kv := a.b.KV()
if kv == nil {
return kvStore{}, errors.New("the bucket is not on the bus yet")
}
return kvStore{kv}, nil
}
func (a awaitedKV) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
s, err := a.kv()
if err != nil {
return nil, 0, false, err
}
return s.Get(ctx, key)
}
func (a awaitedKV) Create(ctx context.Context, key string, value []byte) (uint64, error) {
s, err := a.kv()
if err != nil {
return 0, err
}
return s.Create(ctx, key, value)
}
func (a awaitedKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
s, err := a.kv()
if err != nil {
return 0, err
}
return s.Update(ctx, key, value, revision)
}
func (a awaitedKV) Keys(ctx context.Context) ([]string, error) {
s, err := a.kv()
if err != nil {
return nil, err
}
return s.Keys(ctx)
}
func (a awaitedKV) Delete(ctx context.Context, key string) error {
s, err := a.kv()
if err != nil {
return err
}
return s.Delete(ctx, key)
return catalogue.ParseManifestForInputs([]byte(raw))
}
@@ -13,9 +13,16 @@ import (
"mesh-delivery-checker/internal/check"
)
// The forge's tool the checker reads with (mesh-catalog modules/gitea): gitea_get_file answers {file: {content}}, the text
// of a file at a ref. The checker writes nothing to the forge: the forge bridge sets the statuses (ADR 0484 clause 2).
const getFileTool = "gitea.gitea_get_file"
// The forge's tools the checker reads with (mesh-catalog modules/gitea): gitea_get_file answers {file: {content}}, the text
// of a file at a ref, and gitea_compare answers {total_commits, commits}, what head holds that base does not. The checker
// writes nothing to the forge: the forge bridge sets the statuses (ADR 0484 clause 2).
const (
getFileTool = "gitea.gitea_get_file"
compareTool = "gitea.gitea_compare"
commitsTool = "gitea.gitea_commits"
// gitea_list_branches answers {branches: [{name, commit}]}, every branch with the commit it points at.
branchesTool = "gitea.gitea_list_branches"
)
// notFoundStatus is the 404 of the forge's error, `Gitea API <path>: <status> <body>`.
var notFoundStatus = regexp.MustCompile(`: 404( |$)`)
@@ -62,3 +69,61 @@ func (f forgeReads) File(_ context.Context, repository, path, ref string) (strin
}
return *a.File.Content, nil
}
// OnTrunk says whether the commit is already on the repository's main: main holds it when the commit holds nothing main does
// not.
func (f forgeReads) OnTrunk(_ context.Context, repository, commit string) (bool, error) {
owner, repo := ownerRepo(repository)
raw, err := f.ask(compareTool, map[string]string{"owner": owner, "repo": repo, "base": "main", "head": commit})
if err != nil {
return false, err
}
var a struct {
Total *int `json:"total_commits"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.Total == nil {
return false, fmt.Errorf("the answer of %s is not {total_commits}: %v", compareTool, err)
}
return *a.Total == 0, nil
}
// FullSHA is the whole sha of a commit named by its first digits: the newest commit of the history at that ref.
func (f forgeReads) FullSHA(_ context.Context, repository, commit string) (string, error) {
owner, repo := ownerRepo(repository)
raw, err := f.ask(commitsTool, map[string]any{"owner": owner, "repo": repo, "ref": commit, "limit": 1})
if err != nil {
return "", err
}
var a struct {
Commits []struct {
SHA string `json:"sha"`
} `json:"commits"`
}
if err := json.Unmarshal(raw, &a); err != nil || len(a.Commits) == 0 || len(a.Commits[0].SHA) != 40 || !strings.HasPrefix(a.Commits[0].SHA, commit) {
return "", fmt.Errorf("the answer of %s for %s is not a commit that starts with it: %v", commitsTool, commit, err)
}
return a.Commits[0].SHA, nil
}
// BranchHeads is the commit each branch of the repository points at.
func (f forgeReads) BranchHeads(_ context.Context, repository string) (map[string]string, error) {
owner, repo := ownerRepo(repository)
raw, err := f.ask(branchesTool, map[string]string{"owner": owner, "repo": repo})
if err != nil {
return nil, err
}
var a struct {
Branches *[]struct {
Name string `json:"name"`
Commit string `json:"commit"`
} `json:"branches"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.Branches == nil {
return nil, fmt.Errorf("the answer of %s is not {branches: [{name, commit}]}: %v", branchesTool, err)
}
heads := map[string]string{}
for _, b := range *a.Branches {
heads[b.Name] = b.Commit
}
return heads, nil
}
@@ -29,19 +29,49 @@ func TestTheForgeReadsParseWhatTheForgeAnswered(t *testing.T) { // 1624/AC4
f := forgeReads{call: func(key string, body any) (json.RawMessage, error) {
b, _ := json.Marshal(body)
asked = append(asked, key+" "+string(b))
return capturedAnswer(t, "forge-get-file-go-mod"), nil
switch {
case key == getFileTool:
return capturedAnswer(t, "forge-get-file-go-mod"), nil
case strings.Contains(string(b), "cb54756c3865"):
return capturedAnswer(t, "forge-compare-trunk"), nil
}
return capturedAnswer(t, "forge-compare-branch"), nil
}}
text, err := f.File(context.Background(), "novox/mesh-delivery", "modules/mesh-delivery-builder/go.mod", "main")
if err != nil || !strings.Contains(text, "git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865") {
t.Errorf("file = %q, %v", text, err)
}
if len(asked) != 1 || !strings.Contains(asked[0], `"owner":"novox"`) {
if on, err := f.OnTrunk(context.Background(), "novox/mesh-sdk", "cb54756c3865"); err != nil || !on {
t.Errorf("a commit on the trunk: %v, %v", on, err)
}
if on, err := f.OnTrunk(context.Background(), "novox/mesh-sdk", "9aaa5c80ac5c"); err != nil || on {
t.Errorf("a commit of the branch: %v, %v", on, err)
}
if len(asked) != 3 || !strings.Contains(asked[0], `"owner":"novox"`) || !strings.Contains(asked[2], `"base":"main"`) {
t.Errorf("asked %v", asked)
}
bad := forgeReads{call: func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }}
if _, err := bad.File(context.Background(), "novox/x", "go.mod", "main"); err == nil {
t.Error("an answer with no file was taken")
}
if _, err := bad.OnTrunk(context.Background(), "novox/x", "abc"); err == nil {
t.Error("an answer with no total was taken")
}
}
func TestTheForgeNamesTheWholeShaOfAShortCommit(t *testing.T) { // review D2
f := forgeReads{call: func(key string, body any) (json.RawMessage, error) {
if key != commitsTool {
t.Errorf("called %s", key)
}
return capturedAnswer(t, "forge-commits"), nil
}}
if sha, err := f.FullSHA(context.Background(), "novox/mesh-sdk", "cb54756c3865"); err != nil || sha != "cb54756c3865df5ac9c0dc0481ad9ebe2bf013a1" {
t.Errorf("sha = %q, %v", sha, err)
}
if _, err := f.FullSHA(context.Background(), "novox/mesh-sdk", "deadbeefdead"); err == nil {
t.Error("a commit that does not start with the pin was taken")
}
}
func TestAFileTheRefDoesNotHoldIsNotFoundAndOtherFailuresAreNot(t *testing.T) { // 1845/AC1, AC4
@@ -66,4 +96,10 @@ func TestAForgeReadThatReachesNobodyIsUnheard(t *testing.T) { // 1624 C7: "nats:
if !errors.As(err, &u) {
t.Errorf("err = %v, want it marked as reaching nobody", err)
}
if _, err := f.OnTrunk(context.Background(), "novox/mesh-sdk", "abc"); !errors.As(err, &u) {
t.Errorf("compare: %v", err)
}
if _, err := f.FullSHA(context.Background(), "novox/mesh-sdk", "abc"); !errors.As(err, &u) {
t.Errorf("commits: %v", err)
}
}
@@ -24,8 +24,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
)
@@ -81,16 +79,12 @@ type fakeBuildModules struct {
js jetstream.JetStream
ctx context.Context
conflict atomic.Bool
// hold is a commit whose fetch never answers: its ask stays taken, as one lost in a bus reload (issue 734).
hold atomic.Value
mu sync.Mutex
calls map[string]int // by `<module>.<tool>`
testTool *testing.T
// beside is, by workspace, the siblings the fetch placed in it as `<owner/repo> <commit>`, sorted: a digest echoes them
// (1929/AC2; mesh-sdk buildstep.besideOf), or the level runs whole and keeps nothing.
beside map[string][]string
// names are the nodes' names the last check ask carried, for the names check (ADR 0567 §12).
names []string
}
func (f *fakeBuildModules) count(tool string) int {
@@ -104,9 +98,6 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
var v any
switch {
case strings.HasSuffix(tool, "_fetch"):
if h, _ := f.hold.Load().(string); h != "" && args["commit"] == h {
<-f.ctx.Done()
}
name := "go-fetch-clean"
if f.conflict.Load() {
name = "go-fetch-conflict"
@@ -193,14 +184,7 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.Key
}
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: "n1", Put: put,
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
if a.Check != nil {
f.mu.Lock()
f.names = a.Check.NodeNames
f.mu.Unlock()
}
return buildstep.RunAsk(ctx, steps, a)
}}
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
go func() { _ = p.Pull(f.ctx) }()
}
@@ -314,10 +298,6 @@ func startBed(t *testing.T) *bed {
}
b.tools[module] = kv
}
// What the controller keeps of the messages a consumer gave up on (issue 734).
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: buildrun.DeadLettersStream, Subjects: []string{"mesh.events.dead.>"}}); err != nil {
t.Fatal(err)
}
// The checker's stream of events, which the indexer and the forge bridge hear.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
t.Fatal(err)
@@ -325,8 +305,7 @@ func startBed(t *testing.T) *bed {
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
delivery.TurnsBucket: delivery.TurnsHistory} {
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
if err != nil {
t.Fatal(err)
@@ -343,7 +322,6 @@ func startBed(t *testing.T) *bed {
t.Fatal(err)
}
b.setOnline(t, "n1", true)
b.setNode(t, "n1")
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
b.setHealth(t, "n1", module)
}
@@ -376,20 +354,6 @@ func (b *bed) setOnline(t *testing.T, node string, online bool) {
}
}
// setNode writes the node's record in the node-engine's have, as the node-engine does, so the checker names it in each
// check ask (ADR 0567 §12); with no record the checker asks no check.
func (b *bed) setNode(t *testing.T, node string) {
t.Helper()
data, err := delivery.EncodeNodeRecord(delivery.NodeRecord{V: 1, Node: node})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveNodeKey(node)
if _, err := b.buckets[delivery.HaveBucket].Put(b.ctx, key, data); err != nil {
t.Fatal(err)
}
}
func (b *bed) setHealth(t *testing.T, node, module string) {
t.Helper()
data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
@@ -448,12 +412,11 @@ func pull(repository string, number int, head string) delivery.CandidatePull {
func (b *bed) start(t *testing.T, env map[string]string) {
t.Helper()
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: "n1", Module: "mesh-delivery-checker", User: "n1.mesh-delivery-checker", Password: "pw"})
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: "n1", Module: "mesh-delivery-checker", User: "n1.mesh-delivery-checker", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
// The settle window of a group's heads (check.DefaultSettle) is cut short: each bed's candidate is written whole.
e := map[string]string{"MESH_BROKER_FILE": broker, "MESH_CHECKER_SETTLE": "10ms"}
e := map[string]string{"MESH_BROKER_FILE": broker}
for k, v := range env {
e[k] = v
}
@@ -517,12 +480,6 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || len(v.Machines) != 1 || v.Machines[0] != "n1" {
t.Errorf("composition %+v machines %v", v.Composition, v.Machines)
}
b.fake.mu.Lock()
names := b.fake.names
b.fake.mu.Unlock()
if len(names) != 1 || names[0] != "n1" {
t.Errorf("the check ask named the nodes %v, want [n1] from the have's node record", names)
}
o := b.owed(t, 7, head1, "success")
if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run") {
t.Errorf("owed = %+v", o)
@@ -550,12 +507,9 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
}
}
// TestACheckOrACarryOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every
// package taken from the first's kept verdicts: no test step runs and no second kept key appears. Then, through the real bus
// (novox/hq ADR 0548 §3, §5): a third candidate is not claimed while the gate's merge turn on its repository stands, and is
// claimed when the turn's key is deleted; and a move of the trunk carries the first verdict on the recheck stream by its
// unchanged digests, running no test, the verdict naming the trunk it was carried from.
func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2, ADR 0548 §3, §5
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
b := startBed(t)
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
@@ -572,42 +526,6 @@ func TestACheckOrACarryOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 128
if got := keptCount(t, b); got != kept {
t.Errorf("%d kept keys, was %d", got, kept)
}
// §3: the gate's turn on the repository holds a new check off; its delete claims it.
at := time.Now().UTC()
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: "fix/other", Heads: "0123456789ab", Approved: at, Since: at}}})
if err != nil {
t.Fatal(err)
}
tk, _ := delivery.TurnsKey(repo)
if _, err := b.buckets[delivery.TurnsBucket].Put(b.ctx, tk, turn); err != nil {
t.Fatal(err)
}
time.Sleep(300 * time.Millisecond) // the checker hears the turn
third := b.candidate(t, "feat/z", pull(repo, 9, "3333333333333333333333333333333333333333"))
time.Sleep(time.Second)
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, third); !errors.Is(err, jetstream.ErrKeyNotFound) {
t.Fatalf("verdicts.%s exists (%v) while a merge turn holds its repository", third, err)
}
if err := b.buckets[delivery.TurnsBucket].Delete(b.ctx, tk); err != nil {
t.Fatal(err)
}
b.verdict(t, third, ended)
// §5: the trunk moves; the first verdict is carried by its digests, nothing runs.
tests = b.fake.count("mesh-build-go.build_go_test")
const moved = "abababababababababababababababababababab"
b.setTrunk(t, map[string]string{repo: moved})
c := b.verdict(t, first, func(v delivery.Verdict) bool { return ended(v) && v.Trunk[repo] == moved })
if c.State != delivery.VerdictPass || c.CarriedFrom[repo] != trunk || c.Levels[delivery.LevelUnit].Run != 0 || c.Levels[delivery.LevelIntegration].Run != 0 {
t.Fatalf("the carried verdict = %+v, want a pass carried from %s with no test run", c, trunk[:12])
}
if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
t.Errorf("build_go_test called %d times on the carry, was %d: a carry of unchanged digests runs nothing", got, tests)
}
if o := b.owed(t, 7, head1, "success"); !strings.HasPrefix(o.Text, "pass 111111111111 on abababababab | carried from aaaaaaaaaaaa") {
t.Errorf("the carried status says %q", o.Text)
}
}
func keptCount(t *testing.T, b *bed) int {
@@ -687,17 +605,18 @@ func TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts(t *testing.T) { // N6/
}
}
// TestAMemberNamingAMeshSDKCommitOffTheTrunkIsJudgedAndNotRefused: the go.mod the forge answers names a commit that main does
// not hold; the check is not refused for it, since the build takes the latest published mesh-sdk whatever go.mod names
// (novox/hq ADR 0573 §2, §4; issue 1585 S14, which retires issue 1624's pin check).
func TestAMemberNamingAMeshSDKCommitOffTheTrunkIsJudgedAndNotRefused(t *testing.T) { // 1624/AC2 as ADR 0573 §4 changes it
// TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit: the go.mod the forge answers names a commit that main does not
// hold; the verdict fails naming the member and the commit and no test runs (ADR 0484 clause 9).
func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T) { // 1624/AC2, 1929/AC3
b := startBed(t)
b.stalePin.Store(true)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass {
v := b.verdict(t, key, ended)
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") || !strings.Contains(v.Failed.Words, repo+"#7") {
t.Fatalf("verdict = %+v", v)
}
// 1929/AC3: no assertion that no test ran: one ask runs fetch, digest and tests (ADR 0525 §3).
}
// triesStore is the tries bucket as the tools write it.
@@ -757,49 +676,6 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
t.Errorf("a try wrote %s: %v", name, keys)
}
}
// Issue 734: a try whose ask the build seat gives up on (the controller keeps its copy in DEAD_LETTERS with the ask's
// message id) ends as error saying so, and its head asked again is a new try.
b.fake.hold.Store(head2)
lost, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now())
if err != nil {
t.Fatal(err)
}
steps, err := b.js.KeyValue(b.ctx, delivery.StepsBucket("mesh-build-go"))
if err != nil {
t.Fatal(err)
}
askID := ""
for deadline := time.Now().Add(20 * time.Second); askID == "" && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
keys, _ := steps.Keys(b.ctx)
for _, k := range keys {
if e, err := steps.Get(b.ctx, k); err == nil && strings.Contains(string(e.Value()), `"running"`) {
askID = strings.TrimPrefix(k, "n1.")
}
}
}
if askID == "" {
t.Fatal("the held try's ask was never taken")
}
dead := &nats.Msg{Subject: "mesh.events.dead.SEAT_NODE_BUILD_GO.SEAT_NODE_BUILD_GO_CHECK_worker", Header: nats.Header{}}
dead.Header.Set(buildrun.DeadHeaderMsgID, askID)
dead.Header.Set(buildrun.DeadHeaderDeliveries, "5")
dead.Header.Set(buildrun.DeadHeaderStream, "SEAT_NODE_BUILD_GO")
dead.Header.Set(buildrun.DeadHeaderGaveUp, time.Now().UTC().Format(time.RFC3339Nano))
if _, err := b.js.PublishMsg(b.ctx, dead); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
raw, _ := store.Get(lost)
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
break
}
}
if last.State != check.TryError || !strings.Contains(last.Said, buildrun.LostWords+" after 5 deliveries") {
t.Fatalf("the try whose ask was given up = %+v, want error saying so", last)
}
if again, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head2}, time.Now()); err != nil || again == lost {
t.Errorf("its head asked again answered %q, %v, want a new try", again, err)
}
}
// TestTheCheckerSaysOwedReadableOnTheBus: started, the checker says on its own states subject what it keeps, owed among
@@ -826,8 +702,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
break
}
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") ||
!said.Keeps("asks") || said.Readable("asks") || len(said.States) != 7 {
t.Fatalf("the checker said %+v, want its seven states, asks and hearing its own, with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
t.Fatalf("the checker said %+v, want its five states with kept, owed, previews and verdicts readable (1929/AC4; 1887 A5b)", said)
}
}
@@ -2,18 +2,12 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
)
@@ -31,71 +25,12 @@ type judgeDeps struct {
builds check.BuildStates
// compositions is the assigner's bucket of compositions; nil leaves every composition not run.
compositions check.CompositionReader
// nodes names every node with a record; nil hands no names.
nodes *nodeNames
// predefined is the indexer's kept list of predefined keys; nil reads none.
predefined check.PredefinedList
logf func(format string, args ...any)
}
// keptPredefined reads the indexer's key predefined.list (delivery.PredefinedKept), bound once its state appears.
type keptPredefined struct{ b *process.AwaitedBucket }
func (k keptPredefined) PredefinedList(ctx context.Context) (json.RawMessage, error) {
kv := k.b.KV()
if kv == nil {
return nil, process.ErrNotYet
}
e, err := kv.Get(ctx, delivery.PredefinedListKey)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
var kept delivery.PredefinedKept
if err := json.Unmarshal(e.Value(), &kept); err != nil {
return nil, err
}
return kept.List, nil
}
// nodeNames reads every node's name from the node-engine's have, one key `<node>.node` per node with a record (novox/hq
// ADR 0567 §12), at each ask: a node enrolled since is named in the next check.
type nodeNames struct{ kv jetstream.KeyValue }
// openNodeNames opens the node-engine's have for the nodes' names alone: it is not watched, so no check waits on a node.
func openNodeNames(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) *nodeNames {
return &nodeNames{kv: process.OptionalBucket(ctx, js, delivery.HaveBucket, "no check is asked until it is: the names check needs the nodes' names", logf)}
}
func (n nodeNames) Names(ctx context.Context) ([]string, error) {
if n.kv == nil {
return nil, fmt.Errorf("the bucket %s could not be opened at the checker's start", delivery.HaveBucket)
}
lister, err := n.kv.ListKeysFiltered(ctx, "*."+delivery.HaveNodeWord)
if err != nil {
return nil, err
}
var out []string
for key := range lister.Keys() {
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
out = append(out, node)
}
}
sort.Strings(out)
return out, nil
logf func(format string, args ...any)
}
func newJudge(d judgeDeps) check.Judge {
var nodes interface {
Names(ctx context.Context) ([]string, error)
}
if d.nodes != nil {
nodes = d.nodes
}
return &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Predefined: d.predefined, Logf: d.logf}
return &check.Examiner{Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf}
}
// manifestKV reads a manifest by its digest.
@@ -118,7 +53,6 @@ type repoModules struct {
repoOf map[string]string // by module: its repository
commit map[string]string // by module: the commit its record was computed at (the commit the mesh runs)
typed map[string]string // by module: its manifest digest
uses map[string][]string // by module: the mesh packages it uses (Inputs2.Packages, ADR 0573 §2)
}
// Put takes a change of the indexer's modules bucket in: a module's repository and the languages its manifest names.
@@ -127,7 +61,6 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
if r.repoOf == nil {
r.repoOf, r.typed, r.langs, r.commit = map[string]string{}, map[string]string{}, map[string]map[string]bool{}, map[string]string{}
r.tsPkgs, r.path, r.named = map[string][]string{}, map[string]string{}, map[string]map[string][]string{}
r.uses = map[string][]string{}
}
if deleted {
delete(r.repoOf, key)
@@ -135,7 +68,6 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
delete(r.commit, key)
delete(r.path, key)
delete(r.named, key)
delete(r.uses, key)
r.mu.Unlock()
return
}
@@ -150,10 +82,6 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
r.commit[key] = m.Commit
r.path[key] = m.Path
r.named[key] = m.ArtifactPaths
r.uses[key] = nil
if m.Inputs2 != nil {
r.uses[key] = m.Inputs2.Packages
}
_, known := r.langs[m.Manifest]
r.mu.Unlock()
if m.Manifest == "" || known {
@@ -185,28 +113,6 @@ func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted
r.mu.Unlock()
}
// LatestPackages is the latest published version of each mesh package a module of the mesh uses, and of mesh-sdk's Go
// module always (novox/hq ADR 0573 §2): the trunk merge of the module whose directory holds it, as the indexer's modules
// say. A package no module holds is left out: a check builds it at the version go.mod names.
func (r *repoModules) LatestPackages() delivery.PackageVersions {
r.mu.Lock()
defer r.mu.Unlock()
used := map[string]bool{check.SDKModulePath: true}
held := map[string]delivery.Module{}
for module, repo := range r.repoOf {
for _, p := range r.uses[module] {
used[p] = true
}
held[module] = delivery.Module{Repository: repo, Path: r.path[module], Commit: r.commit[module]}
}
pkgs := make([]string, 0, len(used))
for p := range used {
pkgs = append(pkgs, p)
}
versions, _ := delivery.PackageVersionsOf(pkgs, held)
return versions
}
// PackagePaths are the paths, relative to the repository, that the TypeScript package artifacts of a repository's modules
// are built from, sorted: the named paths the indexer keeps in each module's value (delivery.Module.ArtifactPaths; only the
// indexer reads them from a manifest, the lead's decision L8), under the module's directory. None for a module whose value
@@ -276,20 +182,6 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu
return b, err == nil, err
}
// ManifestDigests is the manifest digest of every module the indexer holds, by module: the catalogue the indexer's judge
// judges a change's manifests in (novox/hq ADR 0578 §2).
func (r *repoModules) ManifestDigests() map[string]string {
r.mu.Lock()
defer r.mu.Unlock()
out := map[string]string{}
for module, digest := range r.typed {
if digest != "" {
out[module] = digest
}
}
return out
}
// CommitOf is the commit the indexer's record of the module names: the commit its fingerprint first appeared at, so the source
// of the build the mesh runs (a later merge that leaves the module alone writes no record, the operator 2026-10-25).
func (r *repoModules) CommitOf(module string) string {
@@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"os"
"reflect"
"strings"
"testing"
@@ -113,20 +112,6 @@ func TestTheTypeScriptPackagesPathsComeFromTheIndexersModulesNotTheManifest(t *t
if got := r.PackagePaths("novox/mesh-delivery"); len(got) != 0 {
t.Errorf("another repository: %v", got)
}
// Issue 1585 (ADR 0573 §2): the latest published version of mesh-sdk's Go module and of every package a module uses is
// the commit of the module that holds it; a package no module holds is left out.
put(".", nil)
const contract = "git.novox.be/novox/mesh-controller/modules/route-proxy/contract"
user, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: strings.Repeat("b", 40), FP: "src1:" + strings.Repeat("b", 64),
Manifest: delivery.ManifestDigest([]byte(sdkManifest)), Path: "modules/x",
Inputs2: &delivery.KeptInputs2{Tree: "t", Packages: []string{contract, "git.novox.be/novox/mesh-sdk/go"}}})
if err != nil {
t.Fatal(err)
}
r.Put(context.Background(), "x", user, false)
if got := r.LatestPackages(); !reflect.DeepEqual(got, delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": strings.Repeat("a", 40)}) {
t.Errorf("latest packages = %v, want mesh-sdk at its module's commit and no unheld contract", got)
}
}
// A manifest whose config value falls back to a predefined key (ADR 0348 §7) is read, not logged as unreadable for the
@@ -37,8 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
if !equal(manifest, announced) {
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
}
// kept is read by the Go build module for a check ask (issue 1887 design limit 1); tries by the delivery view (build_usage).
if want := []string{"kept", "owed", "previews", "tries", "verdicts"}; !equal(readable, want) {
// kept is read by the Go build module for a check ask (issue 1887 design limit 1).
if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
t.Errorf("readable %v, want %v", readable, want)
}
}
@@ -1,50 +0,0 @@
package main
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-checker/internal/check"
)
// lazyTurns is the gate's merge turns as the checker reads them (novox/hq ADR 0548 §3), opened when the gate's bucket is
// there: before each claim it is opened again until it is, so a checker that started before the gate made the bucket holds
// checks from the gate's first turn on, with no timer and no restart. Until then it holds nothing, and says so once, loudly,
// and once again when it opens.
type lazyTurns struct {
turns *check.Turns
open func() error
logf func(format string, args ...any)
mu sync.Mutex
opened bool
said bool
}
// ensure opens the bucket once it is there; a failure is said once.
func (l *lazyTurns) ensure() {
l.mu.Lock()
defer l.mu.Unlock()
if l.opened || l.open == nil {
return
}
if err := l.open(); err != nil {
if !l.said {
l.said = true
l.logf("NO MERGE TURN HOLDS A CHECK: the gate's bucket %s cannot be read (%v); every check is claimed as if no approved merge waited, "+
"and the bucket is opened again before each claim until it is there", delivery.TurnsBucket, err)
}
return
}
l.opened = true
if l.said {
l.logf("the gate's merge turns are read from %s from now on: an approved merge's turn holds checks off its repositories again", delivery.TurnsBucket)
}
}
// Holding opens the bucket when it is not yet, then answers what the turns hold.
func (l *lazyTurns) Holding(repositories []string, heads string) (string, delivery.TurnWait, bool) {
l.ensure()
return l.turns.Holding(repositories, heads)
}
@@ -7,7 +7,12 @@ package main
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
@@ -31,7 +36,7 @@ import (
// eventSubject is the subject of an event of the checker's stream checks (the bus names the stream CHECKS): the indexer's
// watch of the previews bucket and the forge bridge's watch of the owed bucket are woken by them.
func eventSubject(event string) string {
return process.ModuleStreamSubject(delivery.CheckerModule, event)
return "mesh.stream." + delivery.CheckerModule + ".event." + event
}
const (
@@ -48,6 +53,50 @@ func main() {
}
}
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
func readCredential(path string) (credential, error) {
raw, err := os.ReadFile(path)
if err != nil {
return credential{}, err
}
var c credential
if err := json.Unmarshal(raw, &c); err != nil {
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
}
if c.URL == "" || c.User == "" {
return credential{}, errors.New("the broker credential names no bus address or user")
}
return c, nil
}
// pinned accepts exactly the certificate with this SHA-256 and no other.
func pinned(want string) *tls.Config {
return &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
MinVersion: tls.VersionTLS12,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("the bus presented no certificate")
}
sum := sha256.Sum256(rawCerts[0])
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
}
return nil
},
}
}
// durationEnv reads a duration setting from the environment, def when it is not set.
func durationEnv(getenv func(string) string, name string, def time.Duration) (time.Duration, error) {
v := strings.TrimSpace(getenv(name))
@@ -63,33 +112,46 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
// run is the program: it returns when ctx ends or the bus cannot be used.
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate; tries by the delivery view.
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
var keptStates = []announce.State{
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
// Whether DEAD_LETTERS is heard (issue 734): read by the checker's own status tool alone.
{Name: "hearing"},
// The asks each running check put (check.AsksState): read by the checker's own checker_ask_again alone.
{Name: "asks"},
{Name: "kept", Readable: true},
{Name: "owed", Readable: true},
{Name: "previews", Readable: true},
// The tries are read by the delivery view, which tells a try's asks from a candidate's by the asks each names (build_usage).
{Name: "tries", Readable: true},
{Name: "tries"},
{Name: "verdicts", Readable: true},
}
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-checker: ", log.LstdFlags)
cred, err := process.ReadBrokerAccount(getenv)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
}
cred, err := readCredential(path)
if err != nil {
return err
}
conn, js, err := cred.Dial()
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
nats.CustomInboxPrefix("_INBOX." + cred.User),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return err
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
// Its own states, the candidates it checks, and the modules and manifests that say which build modules judge a repository:
// a check without one of them would be wrong or unwritten, so each stops the start, naming its grant (issue 1945).
buckets := map[string]jetstream.KeyValue{}
@@ -101,7 +163,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the
// buckets are read and written on the connection above.
caller, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
account, err := process.ReadAccount(path)
if err != nil {
return err
}
caller, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
}
@@ -120,10 +186,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
settleWindow, err := durationEnv(getenv, "MESH_CHECKER_SETTLE", check.DefaultSettle)
if err != nil {
return err
}
link := strings.TrimSpace(getenv("MESH_CHECKER_LINK"))
if link == "" {
link = defaultLink
@@ -131,37 +193,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
tools := &buildrun.ToolchainSet{}
presence := &buildrun.PresenceSet{}
trunks := &check.Trunks{}
turns := &check.Turns{}
lazy := &lazyTurns{turns: turns, logf: logger.Printf}
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
asks := buildrun.JetStreamAsks{JS: js}
// The asks the build seats gave up on (issue 734): heard from DEAD_LETTERS, each ends the step its asker waits for.
// Heard in the background, and again at a reconnect or an ask while it is not; whether it is heard is kept in the
// state hearing, which checker_status says (dead_letters).
dead := &buildrun.DeadLetters{}
// The state may not be on the bus yet when a delivery started the checker (the grants part creates it, issue 1790): it
// is bound when the bus says it was created, and the latest hearing is written then.
hearingKV := process.AwaitBucket(ctx, js, conn, check.HearingBucket, "checker_status cannot say whether DEAD_LETTERS is heard", logger.Printf)
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, []string{delivery.AskGo, delivery.AskNode}, dead,
logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf))
defer hearing.Stop()
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
heard := buildrun.HearEnds(ctx, asks, asks, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, logger.Printf)
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
waiter := buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}
asksKV := process.AwaitBucket(ctx, js, conn, check.AsksBucket, "checker_ask_again refuses every check, as it cannot tell whether its ask waits in a stream", logger.Printf)
ch := &check.Checker{
Asks: awaitedKV{asksKV},
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
Trunks: trunks,
Turns: lazy,
Bound: checkBound,
Link: link,
Logf: logger.Printf,
Settle: settleWindow,
}
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
// says nothing of the builds, until the checker starts again.
@@ -187,7 +228,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return cw, nil
}}
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
announce := func(ctx context.Context, event, key string) {
body, _ := json.Marshal(map[string]string{"key": key})
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
@@ -195,26 +236,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
}
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
ch.Judge = newJudge(judgeDeps{
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce, compositions: compositions,
nodes: openNodeNames(ctx, js, logger.Printf),
predefined: keptPredefined{process.AwaitBucket(ctx, js, conn, delivery.IndexerModule+"_"+delivery.IndexerPredefinedState,
"a change's manifests are judged with no predefined keys, each fallback as its value declares it", logger.Printf)},
})
ch.Announce = announce
tryer.Judge = ch.Judge
// The toolchains first, so a check that starts from the first candidate finds the nodes.
watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger)
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
// placement is settled first, so the toolchains settle through the gate already knowing it.
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
watch.put = gate.Toolchain
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, ch.NodesChanged, logger.Printf)
assignErrs, err := watchAssignments(ctx, js, gate, ch.NodesChanged, logger)
if err != nil {
return err
}
@@ -226,12 +263,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// module.
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
stream := toolchainStreams[module]
cons, err := cred.EventConsumer(ctx, js, stream)
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
if err != nil {
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the checker starts again: %v", cred.Node, cred.Module, stream, module, err)
continue
}
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { watch.Reported(m) }, logger.Printf)
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
@@ -275,42 +313,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}()
}
}
// The gate's merge turns (novox/hq ADR 0548 §3): no check is claimed of a repository an approved merge waits on, and the
// delete of its turn claims it. Read before the candidates, so no check starts past a turn written before this start.
// Where the gate's bucket is not there yet it is read as empty, and opened again before each claim until it is there
// (lazyTurns), so a gate delivered after the checker started holds checks from its first turn on.
lazy.open = func() error {
kv, err := js.KeyValue(ctx, delivery.TurnsBucket)
if err != nil {
return err
}
tw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { turns.Put(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, tw, func(e jetstream.KeyValueEntry) {
if turns.Put(e.Key(), e.Value(), !opOf(e)) {
ch.Reconsider(ctx) // a turn went or changed: a check it held is claimed now
}
})
}()
return nil
}
lazy.ensure()
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) {
before := repos.CommitOf(check.NodeEngineModule)
repos.Put(ctx, e.Key(), e.Value(), !opOf(e))
if after := repos.CommitOf(check.NodeEngineModule); after != before {
ch.TrunkMoved(ctx) // the node-engine the mesh runs moved: a verdict that read it beside is carried (ADR 0548 §6)
}
})
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
}()
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
@@ -340,21 +349,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
})
}()
// The checker's own verdicts, updates only, in one watch: one checker_ask_again wrote asks its check again
// (Checker.VerdictWritten); a deleted key of heads a candidate holds is judged again (Checker.VerdictDeleted, issue 1733).
vw, err := buckets[delivery.CheckerVerdictsBucket].WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
return fmt.Errorf("the checker's verdicts cannot be watched, so no check can be asked again by hand and a deleted verdict would never be written again: %w", err)
}
go func() {
errs <- follow(ctx, vw, func(e jetstream.KeyValueEntry) {
if opOf(e) {
ch.VerdictWritten(ctx, e.Key(), e.Value())
} else {
ch.VerdictDeleted(ctx, e.Key())
}
})
}()
logger.Printf("started on %s", cred.Node)
go func() { errs <- follow(ctx, w, each) }()
select {
@@ -364,7 +358,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
case err := <-errs:
return err
case err := <-placementErrs:
case err := <-assignErrs:
return err
case err := <-watch.errs:
return err
@@ -7,8 +7,7 @@ import (
)
// 1912: with checks as asks on the build seats' streams, the stream redelivers a lost node's ask (ADR 0525 §6), so the
// checker program watches no node-engine have to start again a check that failed for want of a node. It reads the have's
// keys only for the nodes' names at each ask (judge.go, novox/hq ADR 0567 §12).
// checker program opens no node-engine have to start again a check that failed for want of a node.
func TestTheCheckerWatchesNoHave(t *testing.T) {
raw, err := os.ReadFile("main.go")
if err != nil {
@@ -1,62 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"log"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and
// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
// issue 774).
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
set := &buildrun.ToolchainSet{}
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
@@ -0,0 +1,211 @@
package main
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
// toolchain agreement this file repairs.
import (
"context"
"encoding/json"
"fmt"
"log"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
return false
}
if open {
g.standing[key] = since
} else {
delete(g.standing, key)
}
if !g.known {
return false
}
changed := false
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
return changed
}
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
// passed on changed.
func (g *toolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
if err != nil {
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
changed(ctx)
}
})
}()
return errs, nil
}
@@ -0,0 +1,218 @@
package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
func gateReport(t *testing.T, node, version string) (string, []byte) {
t.Helper()
key, err := delivery.ToolchainKey(node, "typescript")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
if err != nil {
t.Fatal(err)
}
return key, value
}
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
key, value := gateReport(t, node, version)
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(said) != 1 {
t.Fatalf("ace's report is said left out once; said %v", said)
}
// ace reports again while unassigned (an old value seen again): still left out.
key, value := gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
// A report deleted from the bucket goes whether its node is assigned or not.
key, _ = gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
t.Fatal("deleting g14's report passed nothing on")
}
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -14
View File
@@ -1,17 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502 h1:XndvkzyAFNde/821lEfYObqMTURzmve/w9Ks+QrBBJ8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005736-7b09de1e5502/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3 h1:an2f0MLx9OhaDLu5i51OMPBbuGwX+aoTzqOkFLMpVfE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101012031-7cab079e2ec3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c h1:rlUCZ5ifLojqDUTkcCCcPrugTzV2RT6F6JbIVTJkaC8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101013319-25a2cecd989c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236 h1:JEkSnrmHs5QIRJGhtrghLC1+C0AxqMpef8a32d/BLPg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014012-a20bdf795236/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc h1:kVq7sS4I0EnL8DcAA2a3YstnXS6WmYYzJz7eK7IZ9RY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101014310-19c204273bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62 h1:R5C8ICjiaDITHRFR/uIATwARPPXKxssUC70m0svstb0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101020750-62cddf4efc62/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4 h1:05sVel/UOvz4/g0dxrDOeVUL7BGakVrxPBlqRq02ZhM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026095702-79649c3fa9f4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,355 +0,0 @@
package check
import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// One check per delivery (novox/hq ADR 0548): the checker's part. A merge turn holds new checks off its repository (§3);
// a trunk or sibling move never ends a running check and carries its verdict once it ended (§1, §5, §6); only a head git
// refused gets a whole check after a move (§7); the line of a carried verdict says so (§8).
// turnOf is a turn's value on a repository for a delivery of the heads hash.
func turnOf(t testing.TB, repo, group, heads string) []byte {
t.Helper()
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
data, err := delivery.EncodeTurn(delivery.Turn{Repository: repo, Waiting: []delivery.TurnWait{{Group: group, Heads: heads, Approved: at, Since: at}}})
if err != nil {
t.Fatal(err)
}
return data
}
func turnKey(t testing.TB, repo string) string {
t.Helper()
k, err := delivery.TurnsKey(repo)
if err != nil {
t.Fatal(err)
}
return k
}
// waitFor polls a condition the checker's goroutines make true, failing after two seconds.
func waitFor(t testing.TB, what string, ok func() bool) {
t.Helper()
for deadline := time.Now().Add(2 * time.Second); !ok(); time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatalf("%s did not happen", what)
}
}
}
// pullWithFP is a pull request touching the module m at the fingerprint `src2:` and 64 times the hex digit d.
func pullWithFP(repo string, number int, head string, d string) delivery.CandidatePull {
p := pullOf(repo, number, head)
p.Modules = []string{"m"}
p.Fps = map[string]delivery.CandidateModule{"m": {FP2: "src2:" + strings.Repeat(d, 64)}}
return p
}
func TestOneCheckPerDelivery(t *testing.T) {
ctx := context.Background()
for name, run := range map[string]func(t *testing.T){
// §3: no claim while a turn names one of its repositories; a claim on the turn's delete event.
"a turn on its repository holds a new check off until the turn goes": func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 0 || len(r.verdicts.vals) != 0 {
t.Fatalf("judged %d times with a turn on %s, verdicts %v", j.count(), repoA, r.verdicts.vals)
}
if held := r.ch.Held()[keyOf(t, "feat/x", cand)]; !strings.Contains(held, repoA) || !strings.Contains(held, "fix/other") {
t.Errorf("the hold is said as %q, want the turn's repository and group", held)
}
if !turns.Put(turnKey(t, repoA), nil, true) {
t.Fatal("the turn's delete changed nothing")
}
r.ch.Reconsider(ctx) // what the watch of turns does on the delete
r.ch.Wait()
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
t.Errorf("judged %d times after the turn went, want once", j.count())
}
},
"a turn on another repository, or of its own heads, holds nothing": func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
c, _ := delivery.DecodeCandidate(cand)
turns.Put(turnKey(t, repoSDK), turnOf(t, repoSDK, "fix/other", "0123456789ab"), false)
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "feat/x", GateHeads(c)), false)
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 1 {
t.Errorf("judged %d times, want once: the turns hold another repository and its own merge", j.count())
}
},
// §3, the race: a check claimed before the gate's turn is judged on the commit it read, never restarted.
"a check claimed before a turn was written runs to its verdict": func(t *testing.T) {
release, started := make(chan struct{}), make(chan struct{})
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
close(started)
<-release
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
turns := &Turns{}
r.ch.Turns = turns
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
<-started
turns.Put(turnKey(t, repoA), turnOf(t, repoA, "fix/other", "0123456789ab"), false)
r.ch.Reconsider(ctx)
close(release)
r.ch.Wait()
if j.count() != 1 || r.verdict(t, keyOf(t, "feat/x", cand)).State != delivery.VerdictPass {
t.Errorf("judged %d times; the running check is not ended by the turn", j.count())
}
},
// §1, §5: a trunk move during a running check calls the judge once, writes its verdict, keeps the status pending (a pass
// on an old trunk is never shown), then asks one carry, on the recheck stream, and no check.
"a trunk move during a check ends nothing and asks one carry after it": func(t *testing.T) {
var r *rig
var kinds []string
var from []map[string]string
judged := make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
if call == 1 {
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
r.ch.TrunkMoved(ctx) // as the watch of the source says it
close(judged)
} else if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status during the carry is %q, want pending: the pass on the old trunk is never shown", s)
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
<-judged
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || kinds[0] != KindNew || kinds[1] != KindTrunk || from[0] != nil || from[1][repoA] != trunkA {
t.Fatalf("judged %d times as %v carried from %v; want a whole check, then one carry from %s", j.count(), kinds, from, trunkA[:12])
}
if v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
t.Errorf("the verdict is %s on %v carried from %v; want a pass on the new trunk carried from the old", v.State, v.Trunk, v.CarriedFrom)
}
if s := owedState(t, r, repoA, 7, head1); s != "success" {
t.Errorf("the status after the carry is %q, want success", s)
}
if r.verdicts.creates[keyOf(t, "feat/x", cand)] != 1 {
t.Error("the verdict key was made again: a carry rewrites it")
}
},
// §5: the composition is kept when no fingerprint of a module it composed changed, and runs again otherwise.
"a carry keeps a passed composition while the fingerprints stay the same": func(t *testing.T) {
var kept []bool
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
kept = append(kept, c.Composition != nil)
v := passing()
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "v0.1.14-x", FPSet: c.FPSet}
return Outcome{Verdict: v}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "a"))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx) // the indexer's fingerprints of the candidate stay the same
r.ch.Wait()
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullWithFP(repoA, 7, head1, "b"))) // a fingerprint changed with the move
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if len(kept) != 3 || kept[0] || !kept[1] || kept[2] {
t.Errorf("the composition was kept in the checks %v; want composed, kept on the carry, composed again once a fingerprint changed", kept)
}
},
// §6: a sibling's move puts the verdict out of date, its status pending saying it is carried, and the carry's verdict
// names the sibling's old commit; the status follows the carry's verdict. Which packages run is the digest's (only
// those that name MESH_CHECK_BESIDE change with a sibling), the build module's part.
"a sibling's move is carried and the status follows the carry": func(t *testing.T) {
var r *rig
var from map[string]string
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
if call == 2 {
from = c.CarriedFrom
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status during the sibling's carry is %q, want pending", s)
}
}
return Outcome{Verdict: passing(), Beside: map[string]string{catalogueRepository: r.trunks.Heads()[catalogueRepository]}}, nil
}}
r = newRig(t, j)
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS})
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if b := r.verdict(t, keyOf(t, "feat/x", cand)).Beside; b[catalogueRepository] != trunkS {
t.Fatalf("the verdict names the siblings %v, want the catalogue at %s", b, trunkS[:12])
}
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, catalogueRepository: trunkS2})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || from[catalogueRepository] != trunkS || v.CarriedFrom[catalogueRepository] != trunkS || v.Beside[catalogueRepository] != trunkS2 {
t.Errorf("judged %d times, carried from %v, verdict beside %v carried from %v; want one carry over the catalogue's move", j.count(), from, v.Beside, v.CarriedFrom)
}
if s := owedState(t, r, repoA, 7, head1); s != "success" {
t.Errorf("the status after the carry is %q, want success", s)
}
},
// §7: a carry whose merge conflicts fails naming the files and asks no check; a later move onto which the head merges
// cleanly asks a whole check, not a carry.
"a carry git refuses fails naming the files, and the next move asks a whole check": func(t *testing.T) {
var kinds []string
var from []map[string]string
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
kinds, from = append(kinds, c.Kind), append(from, c.CarriedFrom)
if call == 2 {
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "ace",
Words: repoA + "#7 " + ConflictWords + "go.mod"}}}, nil
}
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
r.ch.Reconsider(ctx) // anything else that looks again asks nothing of a conflict
r.ch.Wait()
if v := r.verdict(t, keyOf(t, "feat/x", cand)); j.count() != 2 || !IsConflictFailure(v) || !strings.Contains(v.Failed.Words, "go.mod") {
t.Fatalf("judged %d times, verdict %+v; want the carry failed naming go.mod and nothing asked after it", j.count(), v.Failed)
}
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if j.count() != 3 || kinds[1] != KindTrunk || from[1] == nil || kinds[2] != KindNew || from[2] != nil {
t.Errorf("checks %v carried from %v; want a whole check, a carry, then a whole check after the conflict", kinds, from)
}
},
// §8, §6: the line of a carried verdict names the head on the new trunk commit first, as the gate and the merge tool
// read a line, then where it was carried from and what the carry ran and kept; while it is carried it is pending and says so.
"the line of a carry says where it was carried from": func(t *testing.T) {
cs, p := lineCase(t, pullOf(repoA, 7, head1))
v := passVerdict()
v.Trunk = map[string]string{repoA: trunkA}
v.Beside = map[string]string{catalogueRepository: trunkS2}
v.CarriedFrom = map[string]string{repoA: trunkA2, catalogueRepository: trunkS}
line := Line(v, cs, p, nil, nil, nil)
if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa | carried from abababababab, mesh-catalog 555555555555 | unit 12 run") {
t.Errorf("the carried line = %q", line)
}
if head, trunk := checkWordsOf(line); head != head1[:12] || trunk != trunkA[:12] {
t.Errorf("the line names %s on %s first, want the head on the new trunk", head, trunk)
}
cs.CarriedFrom = map[string]string{repoA: trunkA2}
pending := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil)
if !strings.HasPrefix(pending, "being carried to 111111111111 on aaaaaaaaaaaa | from abababababab") {
t.Errorf("the pending line of a carry = %q", pending)
}
},
// §3 (review round 1): two approved merges in one turn whose trunk moved each get their carry; neither holds the other's.
"two merges waiting in one turn each get their carry": func(t *testing.T) {
var r *rig
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
r = newRig(t, j)
candA, candC := candidateOf(t, "feat/a", pullOf(repoA, 7, head1)), candidateOf(t, "feat/c", pullOf(repoA, 8, head2))
r.ch.Candidate(ctx, "feat%2Fa", candA)
r.ch.Candidate(ctx, "feat%2Fc", candC)
r.ch.Wait()
a, _ := delivery.DecodeCandidate(candA)
c, _ := delivery.DecodeCandidate(candC)
at := time.Date(2026, 10, 26, 9, 0, 0, 0, time.UTC)
turn, err := delivery.EncodeTurn(delivery.Turn{Repository: repoA, Waiting: []delivery.TurnWait{
{Group: "feat/a", Heads: GateHeads(a), Approved: at, Since: at}, {Group: "feat/c", Heads: GateHeads(c), Approved: at.Add(time.Minute), Since: at}}})
if err != nil {
t.Fatal(err)
}
turns := &Turns{}
turns.Put(turnKey(t, repoA), turn, false)
r.ch.Turns = turns
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // a third merge moves the trunk under both
r.ch.TrunkMoved(ctx)
r.ch.Wait()
for _, cand := range [][]byte{candA, candC} {
group := map[bool]string{true: "feat/a", false: "feat/c"}[string(cand) == string(candA)]
if v := r.verdict(t, keyOf(t, group, cand)); v.Trunk[repoA] != trunkA2 || v.CarriedFrom[repoA] != trunkA {
t.Errorf("%s: verdict on %v carried from %v; want its carry claimed and done", group, v.Trunk, v.CarriedFrom)
}
}
},
// §6 (review round 1): the node-engine read at the commit the mesh runs is named, and a new running commit is carried.
"a new running node-engine is carried": func(t *testing.T) {
runs := runsAt{nodeEngineModule: trunkS}
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
return Outcome{Verdict: passing(), Beside: map[string]string{nodeEngineRepo: runs[nodeEngineModule]}}, nil
}}
r := newRig(t, j)
r.ch.Runs = runs
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.ch.TrunkMoved(ctx) // nothing moved: nothing is carried
r.ch.Wait()
runs[nodeEngineModule] = trunkS2
r.ch.TrunkMoved(ctx) // what the indexer's watch does when mesh-host's running commit changes
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if j.count() != 2 || v.CarriedFrom[nodeEngineRepo] != trunkS || v.Beside[nodeEngineRepo] != trunkS2 {
t.Errorf("judged %d times, verdict beside %v carried from %v; want one carry over the node-engine's new running commit", j.count(), v.Beside, v.CarriedFrom)
}
},
// §5: a carry that fails names the move it failed on.
"a carry that fails names the move": func(t *testing.T) {
j := &judgeFunc{f: func(_ context.Context, c *Case, call int) (Outcome, error) {
if call == 1 {
return Outcome{Verdict: passing()}, nil
}
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if v.State != delivery.VerdictFail || v.CarriedFrom != nil || !strings.HasPrefix(v.Failed.Words, "want 4") ||
!strings.Contains(v.Failed.Words, repoA+" "+trunkA[:12]+" to "+trunkA2[:12]) {
t.Errorf("the failed carry is %s carried from %v with words %q; want the test's words, then the move", v.State, v.CarriedFrom, v.Failed.Words)
}
},
} {
t.Run(name, run)
}
}
// checkWordsOf is the head and trunk a line names first, as the gate reads it (mesh-delivery-gate onestatus.go checkLine).
func checkWordsOf(line string) (string, string) {
for _, part := range strings.Split(line, " | ") {
f := strings.Fields(part)
for i := 0; i+2 < len(f); i++ {
if f[i+1] == "on" && len(f[i]) == 12 && len(f[i+2]) == 12 {
return f[i], f[i+2]
}
}
}
return "", ""
}
@@ -1,198 +0,0 @@
package check
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A check is asked again by hand** (the lost check of 2026-10-28; tools-are-how-a-module-is-spoken-to: every recovery
// has a tool). checker_ask_again ends a candidate's running verdict, whose wait judged nothing, with AskedAgainWords and
// the why; the checker hears it on its watch of its verdicts (Checker.VerdictWritten), ends the check of those heads
// running in it, and claims and asks it anew, a whole check (IsEnded). It is refused while a step of the candidate's pull
// requests can still run (checker_status's running_steps): that step's end judges it; and while an ask the check put is
// still in its build seat's stream, taken by no node (it has no step key yet): asking again then would run the check twice
// (issue 1791's kind). The checker keeps the asks each running check put in its state asks (AsksState, by verdict key).
// An ask is not put again under its own id instead: the stream de-duplicates an id within its window, so the copy waiting
// stays and the new one is dropped, and past the window both would run; nothing takes a put ask back (buildrun.Asker).
// AsksState is the checker's state of the asks each running check put, by verdict key (CheckAsks); AsksBucket its bucket.
const (
AsksState = "asks"
AsksBucket = delivery.CheckerModule + "_" + AsksState
)
// CheckAsks are the asks a check put, in order, as the state asks keeps them while it runs; its end deletes the key.
type CheckAsks struct {
Asks []string `json:"asks"`
}
// AsksKV is the state asks: a KV whose key a check's end deletes, so the state holds only the checks that run.
type AsksKV interface {
KV
Delete(ctx context.Context, key string) error
}
// forgetAsks deletes the asks a check put at its end; one that cannot be deleted is said, and the next check of the same
// heads writes the key anew.
func (ch *Checker) forgetAsks(ctx context.Context, key string) {
if err := ch.Asks.Delete(ctx, key); err != nil {
ch.logf("%s: the asks it put could not be forgotten at its end: %v", key, err)
}
}
// recordAsks keeps the asks a check put under its verdict key; nil Asks keeps nothing. A record that cannot be written
// is said: checker_ask_again then refuses the check, as it cannot tell whether its ask waits in a stream.
func (ch *Checker) recordAsks(ctx context.Context, key string, asks []string) {
if ch.Asks == nil {
return
}
data, err := json.Marshal(CheckAsks{Asks: asks})
if err == nil {
_, err = putOnce(ctx, ch.Asks, key, data, func(cur []byte) bool { return string(cur) == string(data) })
}
if err != nil {
ch.logf("%s: the asks it put cannot be kept (%v), so %s refuses it", key, err, AskAgainTool)
}
}
// AskAgainTool asks a candidate's check again by hand.
const AskAgainTool = "checker_ask_again"
// AskedAgainWords open the words of a verdict AskAgainTool wrote; the why follows.
const AskedAgainWords = EndedWords + "asked again by hand (" + AskAgainTool + "): "
// errAskedAgain is the cause a running check is ended with when its verdict was asked again by hand.
var errAskedAgain = errors.New("its verdict was asked again by hand (" + AskAgainTool + ")")
// IsAskedAgain says whether a verdict was written by AskAgainTool.
func IsAskedAgain(v delivery.Verdict) bool {
return IsEnded(v) && strings.HasPrefix(v.Failed.Words, AskedAgainWords)
}
// VerdictEntry is the verdicts state as the ask-again tool writes it: a key's value and revision (nil and 0 for none),
// and a compare-and-set.
type VerdictEntry interface {
Entry(key string) (json.RawMessage, uint64, error)
Update(key string, value []byte, revision uint64) error
}
// AskedAgain is the answer of AskAgainTool.
type AskedAgain struct {
Key string `json:"key"`
Was string `json:"was"`
Said string `json:"said"`
Note string `json:"note"`
}
// AskAgain ends the running verdict of the group's candidate as asked again by hand, saying why. It refuses a group with
// no candidate, a verdict that is not running (a pass or a failure stands until a push or a trunk move), one a step of
// whose pull requests can still run, and one whose asks are not known or one of which no node took yet.
func (r Reader) AskAgain(w VerdictEntry, group, why string) (AskedAgain, error) {
why = strings.TrimSpace(why)
if group == "" || why == "" {
return AskedAgain{}, fmt.Errorf("give the group and why it is asked again")
}
if w == nil {
return AskedAgain{}, fmt.Errorf("the verdicts state cannot be written here")
}
c, ok := r.candidate(group)
if !ok {
return AskedAgain{}, fmt.Errorf("no candidate is held for the group %q", group)
}
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return AskedAgain{}, err
}
raw, rev, err := w.Entry(key)
if err != nil {
return AskedAgain{}, fmt.Errorf("%s cannot be read: %w", key, err)
}
if len(raw) == 0 {
return AskedAgain{}, fmt.Errorf("%s holds no verdict: the checker claims its check when it looks at the candidate (%s)", key, StatusReadTool)
}
v, err := delivery.DecodeVerdict(raw)
if err != nil {
return AskedAgain{}, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
}
if v.State != delivery.VerdictRunning {
return AskedAgain{}, fmt.Errorf("%s is %s, not running: a verdict stands until a push or a trunk move asks it again", key, v.State)
}
steps := r.readSteps()
for _, rs := range steps.running {
if asksOf(c, rs.What) {
return AskedAgain{}, fmt.Errorf("a step of it runs: %s %s on %s since %s (%s); its end judges it", rs.Module, rs.ID, rs.Node, rs.Since, rs.What)
}
}
if r.AskState.Get == nil {
return AskedAgain{}, fmt.Errorf("the checker's asks are not readable here, so whether its ask waits in a build seat's stream cannot be told")
}
kept, err := r.AskState.Get(key)
var put CheckAsks
if err == nil && len(kept) > 0 {
err = json.Unmarshal(kept, &put)
}
if err != nil || len(put.Asks) == 0 {
return AskedAgain{}, fmt.Errorf("the checker names no ask of %s (%v), so whether one waits in a build seat's stream cannot be told; asking again could run it twice", key, err)
}
taken := map[string]bool{}
for _, s := range steps.all {
taken[s.step.ID] = true
}
for _, id := range put.Asks {
if !taken[id] {
return AskedAgain{}, fmt.Errorf("its ask %s waits in its build seat's stream, taken by no node yet: asking again would run it twice; "+
"the seats' backlog is in mesh-delivery-view.build_usage", id)
}
}
words := AskedAgainWords + why
ended := delivery.Verdict{Group: v.Group, Heads: v.Heads, Trunk: v.Trunk, State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Machine: "checker", Words: words}}
data, err := delivery.EncodeVerdict(ended)
if err != nil {
return AskedAgain{}, err
}
if err := w.Update(key, data, rev); err != nil {
return AskedAgain{}, fmt.Errorf("%s changed while it was asked again, so it was not; read it again: %w", key, err)
}
return AskedAgain{Key: key, Was: v.State, Said: words,
Note: "the checker hears it on its watch of its verdicts, ends the wait that judged nothing and asks the check anew, whole; " +
VerdictsReadTool + " shows it running again"}, nil
}
// VerdictWritten takes a change of the checker's verdicts in (its watch, updates only): a verdict AskAgainTool wrote ends
// the check of those heads running in this process, which then considers them again; with none running, the candidate
// that holds them is considered at once. Any other verdict is the checker's own, and left.
func (ch *Checker) VerdictWritten(ctx context.Context, key string, value []byte) {
v, err := delivery.DecodeVerdict(value)
if err != nil || !IsAskedAgain(v) {
return
}
ch.init()
ch.mu.Lock()
if l, ok := ch.live[key]; ok {
l.end(errAskedAgain)
ch.mu.Unlock()
ch.logf("%s: %s; the check running for it is ended and asked again", key, v.Failed.Words)
return
}
var group string
var cand delivery.Candidate
for k, c := range ch.known {
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil && vk == key {
group, cand = k, c
break
}
}
ch.mu.Unlock()
if group == "" {
ch.logf("%s: %s, but no candidate holds these heads, so nothing is asked", key, v.Failed.Words)
return
}
ch.logf("%s: %s; asked again", key, v.Failed.Words)
ch.consider(ctx, group, cand)
}
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -97,19 +96,9 @@ func notJudged(v delivery.Verdict) bool {
return v.Composition != nil && v.Composition.State == delivery.CompositionNotRun && v.Composition.Words == delivery.AssignerNotRunning
}
// unanswered says whether a verdict failed only on its composition, and that only because nodes the change places did
// not answer its placement in time (delivery.CompositionUnanswered, ADR 0559 §5): its tests passed (a composition is read
// only into a passing verdict), so a later composition of the same heads is taken in on its event, with no new check
// (ADR 0548 §1: a part of the mesh not heard is judged again when it is heard).
func unanswered(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Composition != nil && v.Composition.State == delivery.CompositionFail &&
strings.Contains(v.Composition.Words, delivery.CompositionUnanswered) && v.Failed.Words == v.Composition.Words
}
// Composed takes in a composition the assigner wrote after the check's verdict said it was not judged because the
// assigner was not running (the assigner started since), or after it failed only because placed nodes had not answered
// (unanswered): the verdict and its status are written again with it. A composition of heads no candidate holds now, or
// of a verdict that judged its composition otherwise, is left.
// assigner was not running (the assigner started since): the verdict and its status are written again with it. A
// composition of heads no candidate holds now, or of a verdict that judged its composition, is left.
func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdict) {
ch.init()
if comp.State == delivery.CompositionWaits {
@@ -134,21 +123,10 @@ func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdic
return
}
v, err := delivery.DecodeVerdict(cur)
if err != nil {
return
}
switch {
case v.State == delivery.VerdictPass && notJudged(v):
case unanswered(v) && comp.Words != v.Composition.Words:
// The nodes answered since (or another failure stands now): the verdict is its tests' pass with this composition.
v.State, v.Failed = delivery.VerdictPass, delivery.FailedFirst{}
default:
if err != nil || v.State != delivery.VerdictPass || !notJudged(v) {
return
}
withComposition(&v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = cs.FPSet
}
data, err := delivery.EncodeVerdict(v)
if err != nil {
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
@@ -113,27 +113,6 @@ func TestTheAssignerStartingBringsTheComposition(t *testing.T) {
t.Errorf("the status's line says %q", o.Text)
}
}
// ADR 0559 §5: a verdict that failed only because a placed node had not answered takes the same heads' later
// composition in on its event, a pass making it pass with no new check; a composition failing the same way is left.
ch, cs := checkerWithNotJudged(t)
silent := delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionFail, SDK: "x", Machine: "g14",
Words: "g14 (offline) " + delivery.CompositionUnanswered}
ch.Composed(context.Background(), silent)
for _, row := range []struct {
name string
comp delivery.CompositionVerdict
state string
}{
{"the same failure again", silent, delivery.VerdictFail},
{"the node's answer", delivery.CompositionVerdict{Group: cs.Group, Heads: cs.HeadsHash, State: delivery.CompositionPass, SDK: "x"}, delivery.VerdictPass},
} {
ch.Composed(context.Background(), row.comp)
raw, _, _, _ := ch.Verdicts.Get(context.Background(), cs.Key)
if v, err := delivery.DecodeVerdict(raw); err != nil || v.State != row.state || v.Composition == nil || v.Composition.State != row.comp.State {
t.Errorf("%s: the verdict is %+v, %v", row.name, v, err)
}
}
}
// checkerWithNotJudged is a checker holding a candidate whose passing verdict said its composition was not judged.
@@ -12,18 +12,8 @@ type BesideRef struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Ref string `json:"ref,omitempty"`
// atTrunk says the sibling was read at its trunk's head, so a move of that trunk puts the verdict out of date and is
// carried (novox/hq ADR 0548 §6). Never sent: the node-engine at the commit the mesh runs, mesh-sdk at a pin and a
// group's member at its head do not move with a trunk.
atTrunk bool
// runs says the sibling was read at the commit the mesh runs (the node-engine, the indexer's record of mesh-host): a new
// running commit puts the verdict out of date and is carried (ADR 0548 §6). Never sent.
runs bool
}
// NodeEngineModule is the module whose running commit the node-engine sibling is read at.
const NodeEngineModule = nodeEngineModule
const (
catalogueRepository = "novox/mesh-catalog"
nodeEngineRepo = "novox/mesh-host"
@@ -61,27 +51,38 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
switch {
case repo == catalogueRepository:
if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: c.member(repo) == nil})
out = append(out, BesideRef{Repository: repo, Commit: head})
}
case repo == nodeEngineRepo:
commit := ""
if j.Runs != nil {
commit = j.Runs.CommitOf(nodeEngineModule)
}
atTrunk := false
if commit == "" {
commit, atTrunk = heads[repo], true
commit = heads[repo]
}
if commit != "" {
// Named in the verdict either way (ADR 0548 §6, issue 568's own example): at the commit the mesh runs, a new
// running commit puts the verdict out of date; at the trunk, a trunk move does (Checker.besideNow).
out = append(out, BesideRef{Repository: repo, Commit: commit, atTrunk: atTrunk, runs: !atTrunk})
out = append(out, BesideRef{Repository: repo, Commit: commit})
}
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
if err != nil {
return nil, fmt.Errorf("go.mod of %s#%d cannot be read to place the mesh-sdk its tests read: %w", p.Repository, p.Number, err)
}
pin, ok := SDKPin(text)
if !ok {
return nil, fmt.Errorf("go.mod of %s#%d names no mesh-sdk commit, so the mesh-sdk its tests read cannot be placed", p.Repository, p.Number)
}
full, err := j.Forge.FullSHA(ctx, SDKRepository, pin)
if err != nil {
return nil, fmt.Errorf("mesh-sdk %s, which %s#%d pins, cannot be found: %w", pin, p.Repository, p.Number, err)
}
out = append(out, BesideRef{Repository: repo, Commit: full})
default:
if member := c.member(repo); member != nil {
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
} else if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head, atTrunk: true})
out = append(out, BesideRef{Repository: repo, Commit: head})
} else if repo == controllerRepo && p.Repository == nodeEngineRepo {
// The node-engine's step-nine test judges the controller's own manifest and fails without it (issue 1295): a
// checker that cannot place it says so, where placing none left that test to skip.
@@ -107,13 +108,3 @@ func refOf(number int) string {
}
return "refs/pull/" + strconv.Itoa(number) + "/head"
}
// besideAtTrunk are the siblings of refs read at their trunk's head that the case does not merge, by repository: what a
// verdict names as Beside (ADR 0548 §6).
func besideAtTrunk(c *Case, refs []BesideRef, into map[string]string) {
for _, r := range refs {
if _, merged := c.Trunk[r.Repository]; (r.atTrunk || r.runs) && !merged {
into[r.Repository] = r.Commit
}
}
}
@@ -1,42 +0,0 @@
package check
import (
"fmt"
"strings"
)
// A change that adds or modifies a compiled program fails (issue 1577): it is build output, `go build` in a module's folder
// writes one beside go.mod, and `git add -A` takes it. The build module's fetch step, which holds the merged clone, reads the
// first bytes of each changed file and reports the programs (mesh-sdk buildstep.MergeInto); the checker judges the answer.
// NoProgramsReported is the warning of a build module whose fetch answered no programs field: an older module, which cannot
// tell, so the check does not refuse a program it did not look for.
const NoProgramsReported = "the build module did not report programs: a compiled program in the change is not refused"
// checkBinariesOf fails a pull request whose change adds or modifies a compiled program, naming each file. It judges the
// first fetch answer of the pull request, whichever build module gave it; a build module that reports no programs leaves a
// warning on the check and passes it.
func (st *state) checkBinariesOf(t *target) verdictPtr {
key := fmt.Sprintf("%s#%d", t.pull.Repository, t.pull.Number)
if t.fetched.Merge == "" {
return nil // no merge was made: nothing was changed to judge
}
if t.fetched.Programs == nil {
st.c.note(fmt.Sprintf("%s: %s (%s)", key, NoProgramsReported, t.module))
st.c.warn(key, fmt.Sprintf("a compiled program is not refused: %s reported no programs", t.module))
st.j.logf("%s: %s (%s)", key, NoProgramsReported, t.module)
return nil
}
var found []string
for _, p := range *t.fetched.Programs {
if p.Commit != "" {
found = append(found, fmt.Sprintf("%s: a compiled program added in %s and removed later: it stays in history; squash the branch (issue 1577)", p.Path, short12(p.Commit)))
continue
}
found = append(found, p.Path+": a compiled program is build output: remove it and add it to .gitignore (issue 1577)")
}
if len(found) > 0 {
return failed(t.node, fmt.Sprintf("%s: %s", key, strings.Join(found, "; ")))
}
return nil
}
@@ -1,63 +0,0 @@
package check
import (
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func TestAChangeWhoseFetchReportsProgramsFailsNamingEachFile(t *testing.T) { // 1577
prog := func(p ...string) *[]ProgramFile {
out := []ProgramFile{}
for _, f := range p {
path, commit, _ := strings.Cut(f, "@") // `path@commit`: added by a commit of the branch and removed later
out = append(out, ProgramFile{Path: path, Kind: "ELF", Commit: commit})
}
return &out
}
for _, c := range []struct {
name string
programs *[]ProgramFile
fail []string // paths named; none passes
warned bool // the build module reported no programs field
}{
{"none found", prog(), nil, false},
{"one program", prog("modules/m/m"), []string{"modules/m/m"}, false},
{"two programs", prog("a/x", "b/y"), []string{"a/x", "b/y"}, false},
{"added and removed in the branch", prog("gone/prog@0123456789abcdef0123"), []string{"gone/prog: a compiled program added in 0123456789ab and removed later: it stays in history; squash the branch (issue 1577)"}, false},
{"an older build module reports nothing", nil, nil, true},
} {
b := newBench(t)
b.fetch.Merge, b.fetch.Programs = "mmmm", c.programs
cs := caseOf(t, pullOf(repoA, 7, head1))
v := b.judge(t, cs).Verdict
if got := slices.ContainsFunc(cs.Notes(), func(n string) bool { return strings.Contains(n, NoProgramsReported) }); got != c.warned {
t.Errorf("%s: warned = %v, want %v (%v)", c.name, got, c.warned, cs.Notes())
}
line := Line(v, cs, cs.Pulls[0], nil, nil, nil)
if got := strings.Contains(line, "a compiled program is not refused: mesh-build-go reported no programs"); got != c.warned {
t.Errorf("%s: the status line holds the warning = %v, want %v: %s", c.name, got, c.warned, line)
}
if len(c.fail) == 0 {
if v.State != delivery.VerdictPass {
t.Errorf("%s: verdict = %+v", c.name, v)
}
continue
}
if v.State != delivery.VerdictFail {
t.Errorf("%s: verdict = %+v, want a failure", c.name, v)
continue
}
for _, p := range c.fail {
want := p
if !strings.Contains(p, " added in ") {
want = p + ": a compiled program is build output: remove it and add it to .gitignore (issue 1577)"
}
if !strings.Contains(v.Failed.Words, want) {
t.Errorf("%s: words %q do not name %s", c.name, v.Failed.Words, p)
}
}
}
}
@@ -43,43 +43,12 @@ type Case struct {
Try *TryOptions
notesMu sync.Mutex
// warnings are, by pull request, what part of its check did not run (Case.warn).
warnings map[string]string
notes []string
// asks are the ids of the asks the check put on the build seats' streams, in order: a try names them (Try.Asks).
asks []string
// Watch, when set, hears each ask the check puts, as it is put (step nil), and each value its step key takes while
// the check waits for it: the node that took it, the phase it says, its end. A try writes them into its record, so a
// running try names its asks and where each stands (issue 1890). Called from the check's goroutines; it must not block
// for long.
Watch func(askID string, step *delivery.Step)
notes []string
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
Progress *Progress
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
// seats' recheck stream, every other check on check (ADR 0536 §4).
Kind string
// CarriedFrom are, for the carry of a verdict that passed, the trunk and sibling commits it was judged on (ADR 0548 §5);
// nil for a whole check.
CarriedFrom map[string]string
// FPSet is the candidate's fingerprint set (Candidate.FPSet): a composition names the set it judged.
FPSet string
// Composition is, for a carry, the composition that passed on the same fingerprint set: it is kept and not asked again,
// since no fingerprint of a module it composed changed (ADR 0548 §5). Nil composes.
Composition *delivery.Composition
}
// Asked records the id of an ask the check put.
func (c *Case) Asked(id string) {
c.notesMu.Lock()
c.asks = append(c.asks, id)
c.notesMu.Unlock()
}
// Asks are the ids of the asks the check put, in order.
func (c *Case) Asks() []string {
c.notesMu.Lock()
defer c.notesMu.Unlock()
return append([]string(nil), c.asks...)
}
// Progress is the step a check is at, written by the check and read when its bound passes.
@@ -111,11 +80,11 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
if err != nil {
return nil
}
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}, FPSet: c.FPSet}
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
for _, p := range c.Pulls {
switch head, known := trunk[p.Repository]; {
case !known || head == "":
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: %s", p.Repository, p.Number, UntrunkedWordsOf([]string{p.Repository})))
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
default:
cs.Pulls = append(cs.Pulls, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Trunk: head, Modules: p.Modules, Fps: p.Fps})
cs.Trunk[p.Repository] = head
@@ -133,67 +102,6 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
return cs
}
// UntrunkedWords open the words of a verdict, and of a status, of a pull request that cannot be judged because the gate's
// source.main names no trunk head of its repository (issue 1422): never a failure of the change, and judged whole once the
// trunk head is known. It is said loudly, never left silently "new".
const UntrunkedWords = "cannot be judged: the trunk head of "
// UntrunkedWordsOf are those words for the repositories whose trunk head is not known, in order, each named once.
func UntrunkedWordsOf(repos []string) string {
rs := slices.Clone(repos)
slices.Sort(rs)
rs = slices.Compact(rs)
return UntrunkedWords + strings.Join(rs, ", ") + " is not known: the gate's source.main names none, so there is no commit to merge it onto"
}
// IsUntrunked says whether a verdict says its heads cannot be judged for want of a trunk head (UntrunkedWords).
func IsUntrunked(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, UntrunkedWords)
}
// **A candidate outside the mesh is not the checker's** (sweep 2026-10-30 item 7): the forge carries repositories the mesh
// does not deliver (an operator's own), whose pull requests the indexer writes as candidates like any. Their trunk the
// gate never names, so each was judged "cannot be judged" and owed the status error on mesh/check, which the forge
// module wrote 119 times per head into a repository the mesh's forge account may not write (403). A candidate every pull
// request of which is of a repository the gate's source.main names no trunk head of, and touches no module, is outside
// the mesh: judged nothing, owed nothing on the forge, said once in the log and in checker_status (outside_the_mesh). A
// new repository of the mesh is not outside: its first pull request adds a module, and is said untrunked (issue 1422).
// OutsideWords say why a candidate is outside the mesh.
const OutsideWords = "outside the mesh: the gate's source.main names no trunk head of its repositories and it touches no module, so the checker judges nothing and owes no status on the forge"
// Outside says every pull request of c is of a repository trunk names no head of and touches no module. Never with
// trunk unknown (nil): before the gate's source is read nothing is outside.
func Outside(c delivery.Candidate, trunk map[string]string) bool {
if trunk == nil || len(c.Pulls) == 0 {
return false
}
for _, p := range c.Pulls {
if trunk[p.Repository] != "" || len(p.Modules) > 0 || len(p.Fps) > 0 {
return false
}
}
return true
}
// leftOf are the pull requests of a candidate whose repository has no trunk head known, with Trunk empty, in repository and
// number order; nil when every repository's is known.
func leftOf(c delivery.Candidate, trunk map[string]string) []Pull {
var out []Pull
for _, p := range c.Pulls {
if trunk[p.Repository] == "" {
out = append(out, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Modules: p.Modules, Fps: p.Fps})
}
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].Repository != out[j].Repository {
return out[i].Repository < out[j].Repository
}
return out[i].Number < out[j].Number
})
return out
}
// GateHeads is the hash the gate matches the members an Approve covers by (mesh-catalog mesh-delivery-gate onestatus.go
// headsOf): delivery.HeadsHash over every pull request of the candidate, hq's included, as
// `<owner/name lowercase>#<number>=<full head sha lowercase>`. A lone pull request is a group of one. Empty for a candidate
@@ -254,23 +162,6 @@ func (c *Case) note(s string) {
}
}
// warn says of a pull request (pullKey) that a part of its check did not run, in words for its status line (issue 1577).
func (c *Case) warn(key, words string) {
c.notesMu.Lock()
defer c.notesMu.Unlock()
if c.warnings == nil {
c.warnings = map[string]string{}
}
c.warnings[key] = words
}
// Warning is what warn said of a pull request, or "".
func (c *Case) Warning(key string) string {
c.notesMu.Lock()
defer c.notesMu.Unlock()
return c.warnings[key]
}
// Notes are the words noted on the case, in the order they were noted.
func (c *Case) Notes() []string {
c.notesMu.Lock()
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"sort"
"strings"
"sync"
"time"
@@ -43,9 +42,6 @@ type Outcome struct {
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
// that changes none is not named.
Judged map[string]string
// Beside are the siblings the check read at their trunk's head, by repository (ADR 0548 §6): the verdict names them, and
// a move of one is carried.
Beside map[string]string
}
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
@@ -69,28 +65,14 @@ type Checker struct {
Owed KV
Judge Judge
Trunks interface{ Heads() map[string]string }
// Runs says which commit the mesh runs of a module (the indexer's modules): the node-engine a verdict read beside at its
// running commit is out of date when that commit moves (ADR 0548 §6); nil reads every sibling at its trunk.
Runs interface{ CommitOf(module string) string }
// Turns are the gate's merge turns (ADR 0548 §3): no check is claimed of a repository a turn names, but the check of the
// waiting delivery's own heads; nil holds none.
Turns interface {
Holding(repositories []string, heads string) (string, delivery.TurnWait, bool)
}
// Bound is how long one check may run; none when 0 (DefaultBound).
Bound time.Duration
// Asks keeps the asks each running check put, by verdict key (AsksState), for checker_ask_again; nil keeps none.
Asks AsksKV
// Link is the base of the address a status points at; the verdict key follows.
Link string
// Announce says a status was written (EventOwedWritten); nil says nothing.
Announce Announcer
Now func() time.Time
Logf func(format string, args ...any)
// Settle is how long a group's heads must stay the same after the candidate last changed them before its check starts
// (issue 1890): the members of a group are announced one after another, and a check of each partial set is thrown
// away at the next. None when 0; DefaultSettle in the module.
Settle time.Duration
once sync.Once
wg sync.WaitGroup
@@ -101,81 +83,15 @@ type Checker struct {
tried map[string]bool // by verdict key: looked at since this process started
epoch uint64 // changes of the nodes seen since this process started
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
held map[string]string // by verdict key: the turn that holds its check off, as last said
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
settling map[string]*window // by candidates key: the window its heads settle in
// outsideSaid are the candidates outside the mesh said in the log, by group and heads.
outsideSaid map[string]bool
}
// liveCheck is a check running in this process: the candidates key of its group and the end of its run.
type liveCheck struct {
group string
end context.CancelCauseFunc
}
// window is a group's settle window: aborted with errUnsettled when its heads change again or the group goes, and with
// errSettled to end it at once.
type window struct{ abort context.CancelCauseFunc }
var (
errUnsettled = errors.New("the group's heads changed again")
errSettled = errors.New("the group's heads settled")
)
// settleNow ends the settle window of every group at once, as if Settle had passed.
func (ch *Checker) settleNow() {
ch.mu.Lock()
defer ch.mu.Unlock()
for _, w := range ch.settling {
w.abort(errSettled)
}
}
// DefaultSettle is the module's Settle (issue 1890). On 2026-10-26 the three members of a group were announced within 2.8
// seconds of each other (20:31:52.3, :53.7, :55.1 UTC), a session opening or pushing them one API call after another;
// fifteen seconds holds five times that, and costs a check of 10 to 18 minutes under 2.5 % of its time. A member that comes
// later still is no waste beyond its asks already taken: the check of the heads it replaced is ended at once.
const DefaultSettle = 15 * time.Second
// EndedWords open the words of a verdict the checker ended without judging it, its heads no candidate's any more (issue
// 1890): never a failure of the change, and judged whole if a candidate holds those heads again.
const EndedWords = "ended: "
// IsEnded says whether a verdict was ended without being judged (EndedWords).
func IsEnded(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, EndedWords)
}
// superseded is the cause a running check is ended with: the heads that replaced its own, or none when its group went.
type superseded struct{ by string }
func (s superseded) Error() string {
if s.by == "" {
return EndedWords + "no candidate holds these heads any more: the group was closed or merged"
}
return EndedWords + "superseded by " + s.by
}
func (ch *Checker) init() {
ch.once.Do(func() {
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{}
ch.outsideSaid = map[string]bool{}
ch.nodesFail = map[string]uint64{}
})
}
// endOthers ends every check of the group running in this process whose verdict key is not keep: no candidate holds its
// heads any more (issue 1890). Called with ch.mu held.
func (ch *Checker) endOthers(group, keep, by string) {
for k, l := range ch.live {
if l.group == group && k != keep {
l.end(superseded{by: by})
}
}
}
func (ch *Checker) logf(format string, args ...any) {
if ch.Logf != nil {
ch.Logf(format, args...)
@@ -208,91 +124,17 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
ch.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
return
}
vkey, _ := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
ch.mu.Lock()
prev, had := ch.known[key]
ch.known[key] = c
// A check of heads this candidate no longer holds is ended at once (issue 1890): a member's new head, a member joining
// or leaving.
ch.endOthers(key, vkey, GateHeads(c))
moved := !had || prev.HeadsHash != c.HeadsHash
if ch.Settle > 0 && (moved || ch.settling[key] != nil) {
// The group's heads changed: its check waits until they have been the same for Settle, so a group whose members
// are announced one after another is checked once, whole. A later change starts the window again. The wait is a
// context's bound, no timer (ADR 0437 §8).
if w := ch.settling[key]; w != nil {
w.abort(errUnsettled)
}
base, abort := context.WithCancelCause(ctx)
w := &window{abort: abort}
ch.settling[key] = w
ch.wg.Add(1)
go func() {
defer ch.wg.Done()
wait, done := context.WithTimeout(base, ch.Settle)
defer done()
<-wait.Done()
settled := errors.Is(wait.Err(), context.DeadlineExceeded) || errors.Is(context.Cause(wait), errSettled)
ch.mu.Lock()
if !settled || ch.settling[key] != w {
ch.mu.Unlock()
return // changed again, forgotten, or the checker stops
}
delete(ch.settling, key)
now, ok := ch.known[key]
ch.mu.Unlock()
abort(nil)
if ok {
ch.consider(ctx, key, now)
}
}()
ch.mu.Unlock()
return
}
ch.mu.Unlock()
ch.consider(ctx, key, c)
}
// VerdictDeleted takes the delete of a verdict key in (the checker's watch of its verdicts, issue 1733): the gate asks the
// operator's Approve only on a verdict, so heads a candidate holds are never left without one. A check of them running in
// this process writes its verdict anew at its end; otherwise the candidate is considered at once and judged again (or said
// again to be unjudgeable). A key no candidate holds is said and left.
func (ch *Checker) VerdictDeleted(ctx context.Context, key string) {
ch.init()
ch.mu.Lock()
if ch.inflight[key] {
ch.mu.Unlock()
ch.logf("%s: deleted while its check runs; its end writes it anew", key)
return
}
var group string
var cand delivery.Candidate
for k, c := range ch.known {
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil && vk == key {
group, cand = k, c
break
}
}
ch.mu.Unlock()
if group == "" {
ch.logf("%s: deleted, and no candidate holds these heads, so nothing is judged", key)
return
}
ch.logf("%s: deleted; its heads are judged again", key)
ch.consider(ctx, group, cand)
}
// Forget takes a candidate out (its key was deleted): a check of its heads still running is ended, as no candidate holds
// them any more.
// Forget takes a candidate out (its key was deleted).
func (ch *Checker) Forget(key string) {
ch.init()
ch.mu.Lock()
delete(ch.known, key)
if w := ch.settling[key]; w != nil {
w.abort(errUnsettled)
delete(ch.settling, key)
}
ch.endOthers(key, "", "")
ch.mu.Unlock()
}
@@ -329,71 +171,38 @@ func (ch *Checker) heads() map[string]string {
}
func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) {
heads := ch.heads()
if Outside(c, heads) {
ch.outside(ctx, c)
return
}
if left := leftOf(c, heads); len(left) > 0 {
// A pull request the checker cannot claim (its repository's trunk head is not known) leaves its whole group
// unjudged, said in a failure verdict (issue 1422, issue 1733): the gate asks the Approve only on a verdict, and a
// pass of the other members would let the group merge with that one never checked.
ch.untrunked(ctx, c, left)
return
}
cs := CaseOf(c, heads)
cs := CaseOf(c, ch.heads())
if cs == nil {
return // no pull request at all: nothing to judge
return // nothing to judge: no trunk known
}
cs.Queued = ch.now()
ch.mu.Lock()
if ch.settling[key] != nil {
ch.mu.Unlock()
return // its heads are settling: the window's end considers it (Settle)
}
if ch.inflight[cs.Key] {
ch.again[cs.Key] = true // looked at once more when the check ends, so a change of the trunk is never lost
ch.mu.Unlock()
return
}
ch.inflight[cs.Key] = true
run, end := context.WithCancelCause(ctx)
ch.live[cs.Key] = liveCheck{group: key, end: end}
ch.mu.Unlock()
ch.wg.Add(1)
go func() {
defer ch.wg.Done()
vkey := cs.Key
defer func() {
ch.mu.Lock()
delete(ch.inflight, vkey)
delete(ch.again, vkey)
delete(ch.live, vkey)
ch.mu.Unlock()
again := errors.Is(context.Cause(run), errAskedAgain)
end(nil)
if again && ctx.Err() == nil {
ch.consider(ctx, key, ch.candidate(key, c)) // asked again by hand (AskAgainTool): claimed anew
}
}()
for {
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
// ADR 0536 §4). The checker holds no place and no capacity of its own.
ch.prepare(ctx, cs)
ch.try(ctx, run, key, cs)
cs.Kind = ch.kindOf(ctx, cs)
ch.try(ctx, key, cs)
ch.mu.Lock()
more := ch.again[vkey]
delete(ch.again, vkey)
ch.mu.Unlock()
if !more || run.Err() != nil {
more := ch.again[cs.Key]
delete(ch.again, cs.Key)
if !more || ctx.Err() != nil {
delete(ch.inflight, cs.Key)
ch.mu.Unlock()
return
}
// Looked at again for the same heads only: a candidate that moved to other heads has its own check (issue 1890).
ch.mu.Unlock()
if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil {
if next.Key != vkey {
return
}
cs = next
}
cs.Queued = ch.now()
@@ -401,236 +210,30 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
}()
}
// untrunked says, loudly, that pull requests of a candidate cannot be judged because the gate's source.main names no trunk
// head of their repository (issue 1422): its verdict key holds a failure that says so, whatever it held before (a running
// verdict left by a check that can no longer run, or a verdict judged on a trunk that is no longer named: issue 1733), which
// checker_verdicts and checker_status show; and every pull request of the candidate gets the status error with
// UntrunkedWords, the members whose trunk is known too, as their group is not judged. It is logged once per head: when a
// status or the verdict is written, never at a look that finds them already said. Once the trunk head is known the verdict
// is stale (it names no trunk) and the heads are judged whole (prepare, claim).
func (ch *Checker) untrunked(ctx context.Context, c delivery.Candidate, left []Pull) {
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
ch.logf("candidates of %s: no verdict key for heads %q, so it cannot be said that they cannot be judged: %v", c.Group, c.HeadsHash, err)
return
}
repos := make([]string, 0, len(left))
for _, p := range left {
repos = append(repos, p.Repository)
}
words := UntrunkedWordsOf(repos)
v := delivery.Verdict{Group: c.Group, Heads: GateHeads(c), Trunk: map[string]string{}, State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Machine: "checker", Words: words}}
data, err := delivery.EncodeVerdict(v)
if err != nil {
ch.logf("%s: %s, and the verdict cannot say so: %v", key, words, err)
return
}
said, err := putOnce(ctx, ch.Verdicts, key, data, func(cur []byte) bool {
held, err := delivery.DecodeVerdict(cur)
return err == nil && IsUntrunked(held) && held.Failed.Words == words // the same words stand; anything else is replaced
})
if err != nil {
ch.logf("%s: %s, and the verdict cannot say so: %v", key, words, err)
}
for _, cp := range c.Pulls {
okey, err := delivery.CheckerOwedKey(cp.Repository, cp.Number, cp.Head.Head)
if err != nil {
ch.logf("%s: %v", key, err)
continue
}
o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "error",
Target: strings.TrimSuffix(ch.Link, "/") + "/" + key, Text: clipLine(words, MaxLine)}
data, err := delivery.EncodeCheckerOwed(o)
if err != nil {
ch.logf("%s: the status of %s cannot be written: %v", key, okey, err)
continue
}
wrote, err := putOnce(ctx, ch.Owed, okey, data, func(cur []byte) bool {
held, err := delivery.DecodeCheckerOwed(cur)
o.V = delivery.Version
return err == nil && delivery.SameOwed(held, o)
})
if err != nil {
ch.logf("%s: the status of %s cannot be written: %v", key, okey, err)
continue
}
if wrote {
said = true
if ch.Announce != nil {
ch.Announce(ctx, EventOwedWritten, okey)
}
}
}
if said {
names := make([]string, 0, len(left))
for _, p := range left {
names = append(names, fmt.Sprintf("%s#%d@%s", p.Repository, p.Number, short12(p.Head)))
}
ch.logf("%s: %s %s; the verdict and every status of its group say so (error), and it is judged once the trunk head is known", key, strings.Join(names, ", "), words)
}
}
// outside takes a candidate outside the mesh (case.go Outside): no verdict, and every status owed for its pull requests
// taken away, so the forge module stops writing one into a repository the mesh's forge account may not write. Said once
// per heads in the log.
func (ch *Checker) outside(ctx context.Context, c delivery.Candidate) {
var names []string
for _, p := range c.Pulls {
names = append(names, fmt.Sprintf("%s#%d@%s", p.Repository, p.Number, short12(p.Head.Head)))
key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head.Head)
if err != nil {
continue
}
_, _, found, err := ch.Owed.Get(ctx, key)
if err != nil || !found {
continue
}
d, ok := ch.Owed.(Deleter)
if !ok {
continue
}
if err := d.Delete(ctx, key); err != nil {
ch.logf("%s: the status owed of a pull request outside the mesh could not be taken away: %v", key, err)
}
}
said := c.Group + "@" + c.HeadsHash
ch.mu.Lock()
first := !ch.outsideSaid[said]
ch.outsideSaid[said] = true
ch.mu.Unlock()
if first {
ch.logf("candidates of %s: %s %s", c.Group, strings.Join(names, ", "), OutsideWords)
}
}
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
// §5), on the recheck stream, pulled after.
// the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after.
const (
KindNew = "new"
KindNodes = "nodes"
KindTrunk = "trunk"
)
// prepare reads the case's verdict key and says what its check is (ADR 0548 §1, §5, §7): none (or one left running) is a
// head never judged, a whole check; a failure of the nodes is asked again after the nodes changed; a verdict judged on trunk
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing
// to do.
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
// kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
// failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
// move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
if err != nil || !found {
return
return KindNew
}
v, err := delivery.DecodeVerdict(value)
switch {
case err != nil || v.State == delivery.VerdictRunning:
return
return KindNew
case IsNodesFailure(v):
cs.Kind = KindNodes
return
case IsConflictFailure(v):
return // a head git refused: its next check is whole (ADR 0548 §7)
case IsEnded(v), IsUntrunked(v):
return // ended unjudged when its heads went (issue 1890), or never judged for want of a trunk head (issue 1422): checked whole
return KindNodes
}
cs.Kind = KindTrunk
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
cs.CarriedFrom = judgedOn(v)
if c := v.Composition; c != nil && c.State == delivery.CompositionPass && c.FPSet != "" && c.FPSet == cs.FPSet {
kept := *c
cs.Composition = &kept
}
}
}
// stale says whether a verdict was judged on commits that are no longer the case's: a trunk it merged moved, or a sibling it
// read at its trunk's head (ADR 0548 §6).
func (ch *Checker) stale(v delivery.Verdict, cs *Case) bool {
return !sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, ch.besideNow())
}
// besideNow are the commits a sibling is read at now, by repository: its trunk head, and for the node-engine the commit the
// mesh runs when the indexer names one, as besideFor places them.
func (ch *Checker) besideNow() map[string]string {
return besideAt(ch.heads(), ch.Runs)
}
// besideAt places the commits a sibling is read at: the trunk heads, with the node-engine at the commit the mesh runs when
// runs names one (ADR 0548 §6). The checker's decision and the status read the same commits through it.
func besideAt(now map[string]string, runs interface{ CommitOf(module string) string }) map[string]string {
if runs != nil {
if c := runs.CommitOf(nodeEngineModule); c != "" {
out := make(map[string]string, len(now)+1)
for r, h := range now {
out[r] = h
}
out[nodeEngineRepo] = c
return out
}
}
return now
}
// besideMoved says whether a sibling read at these commits moved: the commit it is read at now is known and another.
func besideMoved(beside, now map[string]string) bool {
for r, c := range beside {
if h := now[r]; h != "" && h != c {
return true
}
}
return false
}
// judgedOn are the trunk and sibling commits a verdict was judged on, by repository.
func judgedOn(v delivery.Verdict) map[string]string {
out := make(map[string]string, len(v.Trunk)+len(v.Beside))
for r, c := range v.Beside {
out[r] = c
}
for r, c := range v.Trunk {
out[r] = c
}
return out
}
// carried is what a carry's verdict names as carried from (ADR 0548 §5): the commits it was judged on before, of the
// repositories the new verdict still judges or reads; nil when none of them moved.
func carried(from map[string]string, v delivery.Verdict) map[string]string {
out, moved := map[string]string{}, false
for r, c := range from {
now, ok := v.Trunk[r]
if !ok {
now, ok = v.Beside[r]
}
if ok && now != "" && c != "" {
out[r] = c
moved = moved || now != c
}
}
if !moved {
return nil
}
return out
}
// moveWords say a carry's moves, for the words of a failure: "carried from <old12> to <new12> of <repository>", each moved
// repository, sorted.
func moveWords(from map[string]string, v delivery.Verdict) string {
var moves []string
for r, c := range from {
now, ok := v.Trunk[r]
if !ok {
now = v.Beside[r]
}
if now != "" && now != c {
moves = append(moves, fmt.Sprintf("%s %s to %s", r, short12(c), short12(now)))
}
}
sort.Strings(moves)
return strings.Join(moves, ", ")
return KindTrunk
}
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
@@ -662,9 +265,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return nil, fmt.Errorf("%s: %w", cs.Key, err)
}
if !found {
if ch.heldByTurn(cs) {
return nil, nil
}
rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
if errors.Is(err, ErrConflict) {
return nil, nil // another checker made it first
@@ -685,15 +285,15 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return &claimed{rev}, nil
}
return nil, nil
case IsEnded(cur), ch.stale(cur, cs):
retake = true // heads held again after their check was ended unjudged (issue 1890), or a trunk that moved
case !sameTrunk(cur.Trunk, cs.Trunk):
retake = true
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
ch.mu.Lock()
at, ok := ch.nodesFail[cs.Key]
ch.mu.Unlock()
retake = !ok || epoch > at
}
if !retake || ch.heldByTurn(cs) {
if !retake {
return nil, nil
}
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
@@ -706,59 +306,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return &claimed{rev}, nil
}
// heldByTurn says whether a merge turn holds the case's check off (ADR 0548 §3): an approved merge waits on one of the
// repositories it merges heads into, and it is not the merge of the case's own heads. The check is claimed when the turn's
// key goes (Reconsider on the turns watch). Said once per turn in the log. A check already claimed is never held: one
// claimed in the moment before the gate wrote its turn is judged on the commit it read and carried once after.
func (ch *Checker) heldByTurn(cs *Case) bool {
if ch.Turns == nil {
return false
}
repos := make([]string, 0, len(cs.Trunk))
for r := range cs.Trunk {
repos = append(repos, r)
}
repo, w, held := ch.Turns.Holding(repos, cs.Heads)
why := ""
if held {
why = repo + " " + w.Group + " " + w.Heads
}
ch.mu.Lock()
said := ch.held[cs.Key]
if why == "" {
delete(ch.held, cs.Key)
} else {
ch.held[cs.Key] = why
}
ch.mu.Unlock()
if held && said != why {
ch.logf("%s: not claimed: the approved merge of %s (heads %s) holds a turn on %s, so its check waits for that merge (ADR 0548 §3)",
cs.Key, w.Group, w.Heads, repo)
}
return held
}
// Held is every check a merge turn holds off now, by verdict key, with the turn's repository, group and heads.
func (ch *Checker) Held() map[string]string {
ch.init()
ch.mu.Lock()
defer ch.mu.Unlock()
out := make(map[string]string, len(ch.held))
for k, v := range ch.held {
out[k] = v
}
return out
}
// ConflictWords are what the words of a head that git would not merge onto its trunk say before the files in conflict.
const ConflictWords = "does not merge into the trunk: files in conflict: "
// IsConflictFailure says whether a verdict failed because its head did not merge onto its trunk (ADR 0548 §7): it has no
// pass to carry, so its next check, at a push or at a later trunk move, is a whole check of its head.
func IsConflictFailure(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, ConflictWords)
}
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
// seats' streams (ADR 0525 §6, issue 1912).
@@ -766,13 +313,9 @@ func IsNodesFailure(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, UnheardWords)
}
// try claims the check, judges it within its bound under run and writes the verdict and the owed statuses. A check that ends
// because ctx did is left running: the checker takes it again when it starts. A check whose run was ended because no
// candidate holds its heads any more (issue 1890) is ended: its verdict says so, and no status is written.
func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
if run.Err() != nil {
return // ended before it was claimed: nothing to write
}
// try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends
// because ctx did is left running: the checker takes it again when it starts.
func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
ch.mu.Lock()
first := !ch.tried[cs.Key]
ch.tried[cs.Key] = true
@@ -787,58 +330,13 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
return
}
began := ch.now()
if cs.Watch == nil && ch.Asks != nil {
cs.Watch = func(_ string, step *delivery.Step) {
if step == nil { // put: kept before its wait starts
ch.recordAsks(ctx, cs.Key, cs.Asks())
}
}
defer ch.forgetAsks(ctx, cs.Key)
}
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
out := ch.judge(run, cs)
// An ask the build seat gave up on, or whose step the build module's next process ended because it was replaced under
// it, judged nothing (issue 734, IsLostAsk): the check is asked again under the same claim, its status pending, and after
// LostAskTries it fails as not heard, judged again when the nodes change. A step that ended while its watch was deaf is
// read at the next nudge (HeardEnds) and judged.
for try := 1; IsLostAsk(out.Verdict) && run.Err() == nil; try++ {
if try >= LostAskTries {
out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) +
" times and given up each time: " + UnheardWords)
break
}
ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries)
out = ch.judge(run, cs)
}
if ctx.Err() != nil || errors.Is(context.Cause(run), errAskedAgain) {
return // the checker stops, or the verdict was asked again by hand, which wrote it: its next check claims it anew
}
var why superseded
if errors.As(context.Cause(run), &why) {
ch.end(ctx, cs, cl, why)
out := ch.judge(ctx, cs)
if ctx.Err() != nil {
return
}
v := out.Verdict
v.Group, v.Heads, v.Trunk, v.Beside = cs.Group, cs.Heads, cs.Trunk, nil
for r, c := range out.Beside {
if _, merged := cs.Trunk[r]; !merged && c != "" {
if v.Beside == nil {
v.Beside = map[string]string{}
}
v.Beside[r] = c
}
}
if cs.CarriedFrom != nil {
// A carry (ADR 0548 §5): a pass names the commits it was carried from; a failure names the move it failed on.
switch v.State {
case delivery.VerdictPass:
v.CarriedFrom = carried(cs.CarriedFrom, v)
case delivery.VerdictFail:
if moves := moveWords(cs.CarriedFrom, v); moves != "" {
v.Failed.Words = strings.TrimSpace(v.Failed.Words + "\nfailed on the carry over the move of " + moves + ": the move changed what these tests read")
}
}
}
v.Group, v.Heads, v.Trunk = cs.Group, cs.Heads, cs.Trunk
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
}
@@ -852,50 +350,25 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
}
}
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil {
if _, _, found, gerr := ch.Verdicts.Get(ctx, cs.Key); gerr == nil && !found {
_, err = ch.Verdicts.Create(ctx, cs.Key, data) // its key was deleted while it ran (issue 1733): written anew
}
if err != nil {
ch.logf("%s: the verdict could not be written: %v", cs.Key, err)
return
}
ch.logf("%s: the verdict could not be written: %v", cs.Key, err)
return
}
if IsNodesFailure(v) {
ch.mu.Lock()
ch.nodesFail[cs.Key] = epoch
ch.mu.Unlock()
}
if ch.movedSince(cs) || besideMoved(v.Beside, ch.besideNow()) {
if ch.movedSince(cs) {
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
// check of the new trunk, which Reconsider starts, ends.
ch.owePending(ctx, cs, v)
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending until it is carried", cs.Key, v.State)
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
return
}
ch.writeOwed(ctx, cs, v, out)
ch.logf("%s: %s", cs.Key, v.State)
}
// end writes the verdict of a check whose heads no candidate holds any more (issue 1890), by compare-and-set on its claim,
// and says it once in the log with the asks it had put: they stay in the build seats' streams, since an ask once put cannot
// be taken back (buildrun.Asker has no withdraw), and are no longer waited for. No status is written: the heads that
// replaced these have their own check, whose pending status stands.
func (ch *Checker) end(ctx context.Context, cs *Case, cl *claimed, why superseded) {
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, Trunk: cs.Trunk, State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Machine: "checker", Words: why.Error()}}
data, err := delivery.EncodeVerdict(v)
if err == nil {
_, err = ch.Verdicts.Update(ctx, cs.Key, data, cl.rev)
}
if err != nil {
ch.logf("%s: %s, but the verdict could not say so: %v", cs.Key, why.Error(), err)
return
}
stage, _ := cs.Progress.Now()
asks := cs.Asks()
ch.logf("%s: %s; it was at %q with %d asks put %v, which are no longer waited for", cs.Key, why.Error(), stage, len(asks), asks)
}
// judge runs the case within the bound. A judge that fails to judge, and a check that ran past its bound, are verdicts of
// failure naming the machine and the step.
func (ch *Checker) judge(ctx context.Context, cs *Case) Outcome {
@@ -994,7 +467,7 @@ func (ch *Checker) Sweep(ctx context.Context) error {
continue
}
cur.State = delivery.VerdictFail
cur.Failed = delivery.FailedFirst{Machine: "checker", Words: EndedWords + "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"}
cur.Failed = delivery.FailedFirst{Machine: "checker", Words: "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"}
data, err := delivery.EncodeVerdict(cur)
if err != nil {
return err
@@ -1029,37 +502,14 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
if err != nil || !found {
continue // never judged: no status to put out of date
}
v, err := delivery.DecodeVerdict(value)
if err == nil && !ch.stale(v, cs) {
if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
continue
}
if err == nil && v.State == delivery.VerdictPass {
cs.CarriedFrom = judgedOn(v) // the status says it is being carried to the new commit (ADR 0548 §6)
}
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
}
ch.Reconsider(ctx)
}
// owePending owes mesh/check pending on the heads of a verdict judged on commits that moved while it ran: a pass says it is
// being carried to the trunk's head now (ADR 0548 §6), anything else that it is checked again.
func (ch *Checker) owePending(ctx context.Context, judged *Case, v delivery.Verdict) {
next := judged
ckey, err := delivery.CandidatesKey(v.Group)
ch.mu.Lock()
c, ok := ch.known[ckey]
ch.mu.Unlock()
if err == nil && ok {
if now := CaseOf(c, ch.heads()); now != nil && now.Key == judged.Key {
next = now
}
}
if v.State == delivery.VerdictPass {
next.CarriedFrom = judgedOn(v)
}
ch.writeOwed(ctx, next, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
}
// movedSince says whether a trunk the case was judged on is no longer the trunk's head.
func (ch *Checker) movedSince(cs *Case) bool {
now := ch.heads()
@@ -3,7 +3,6 @@ package check
import (
"context"
"encoding/json"
"fmt"
"strings"
"sync"
"testing"
@@ -48,13 +47,6 @@ func pullOf(repo string, number int, head string) delivery.CandidatePull {
return delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: head}, Modules: []string{}}
}
// newRepoPullOf is a pull request of a new repository of the mesh: its trunk not named yet, its first module added.
func newRepoPullOf(repo string, number int, head string) delivery.CandidatePull {
p := pullOf(repo, number, head)
p.Modules = []string{"mod-u"}
return p
}
// judgeFunc is a Judge written as a function; it counts its calls.
type judgeFunc struct {
mu sync.Mutex
@@ -246,119 +238,14 @@ func TestAnUnheardForgeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *t
}
}
// Issue 1422: a pull request of a repository the gate's source.main names no trunk head of (one never merged through the
// gate) is never dropped silently. It is not judged, its verdict says it cannot be judged and why, its status is error with
// those words, checker_status names it, and the log says it once per head. Once its trunk head is known it is judged whole;
// in a group, the members whose trunk is known are judged and the others' status says why they are not.
func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testing.T) { // N6/AC6, ADR 0484 clause 11, 1422
func TestARepositoryWithNoTrunkGetsNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clause 11
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
var logged []string
r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
const unknown = "novox/unknown"
cand := candidateOf(t, "feat/y", newRepoPullOf(unknown, 4, head1))
key := keyOf(t, "feat/y", cand)
r.ch.Candidate(ctx, "feat/y", cand)
r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf("novox/unknown", 4, head1)))
r.ch.Wait()
r.ch.Reconsider(ctx) // looked at again: nothing is written or said twice
r.ch.NodesChanged(ctx)
r.ch.Wait()
v := r.verdict(t, key)
if j.count() != 0 || !IsUntrunked(v) || !strings.Contains(v.Failed.Words, "the trunk head of "+unknown+" is not known") {
t.Fatalf("judged %d times, verdict %+v, want none judged and a verdict that says it cannot be judged", j.count(), v)
if j.count() != 0 || len(r.verdicts.vals) != 0 || len(r.owed.vals) != 0 {
t.Errorf("judged %d times, %d verdicts, %d statuses, want none", j.count(), len(r.verdicts.vals), len(r.owed.vals))
}
if s := owedState(t, r, unknown, 4, head1); s != "error" {
t.Errorf("%s#4 owes %q, want error", unknown, s)
}
if n := countWith(logged, "cannot be judged"); n != 1 {
t.Errorf("said %d times in the log, want once for the head: %q", n, logged)
}
reader := func() Reader {
ckey, _ := delivery.CandidatesKey("feat/y")
return Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}
}
st, err := reader().Status()
if err != nil {
t.Fatal(err)
}
if why := st.CannotBeJudged["feat/y"]; !strings.Contains(why, unknown) || len(st.Waiting[KindNew]) != 0 || len(st.Unjudged) != 0 {
t.Errorf("cannot_be_judged %v, waiting %v, unjudged %v: want feat/y named as cannot be judged, not as new", st.CannotBeJudged, st.Waiting, st.Unjudged)
}
if rows, _, err := reader().Verdicts("feat/y"); err != nil || len(rows) != 1 || !strings.Contains(rows[0].Reports[unknown+"#4"], "cannot be judged") {
t.Errorf("checker_verdicts %+v (%v), want the report of %s#4 to say it cannot be judged", rows, err, unknown)
}
// Its trunk head becomes known: the heads are judged whole, the same key rewritten, the status success.
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, unknown: trunkA2})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if v := r.verdict(t, key); j.count() != 1 || v.State != delivery.VerdictPass || v.Trunk[unknown] != trunkA2 {
t.Errorf("judged %d times, verdict %+v, want one pass on %s", j.count(), v, trunkA2)
}
if s := owedState(t, r, unknown, 4, head1); s != "success" {
t.Errorf("%s#4 owes %q once its trunk is known, want success", unknown, s)
}
// Issue 1733: the gate asks the Approve only on a verdict, so no candidate is left without one. A group with a member
// whose trunk is not known is not judged in part: its verdict fails naming that repository, and every status says so; a
// running verdict left for heads that cannot be judged is replaced; a verdict key deleted is judged again on its delete.
for _, row := range []struct {
name string
pulls []delivery.CandidatePull
running bool // a running verdict is left under the key first
deleted bool // the key is deleted after its verdict
judged int
want string // the verdict's state, and for a failure the repository it names
}{
{name: "a member the checker cannot claim", pulls: []delivery.CandidatePull{pullOf(repoA, 9, head2), pullOf("novox/other", 2, head2)}, want: "novox/other"},
{name: "a running verdict left for heads that cannot be judged", pulls: []delivery.CandidatePull{newRepoPullOf(unknown, 5, head2)}, running: true, want: unknown},
{name: "a verdict key deleted", pulls: []delivery.CandidatePull{pullOf(repoA, 5, head2)}, deleted: true, judged: 2, want: delivery.VerdictPass},
} {
t.Run(row.name, func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
cand := candidateOf(t, "feat/z", row.pulls...)
key := keyOf(t, "feat/z", cand)
if row.running {
group, heads, _ := delivery.ParseCheckerVerdictKey(key)
raw, _ := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, State: delivery.VerdictRunning})
_, _ = r.verdicts.Create(ctx, key, raw)
}
r.ch.Candidate(ctx, "feat%2Fz", cand)
r.ch.Wait()
if row.deleted {
_ = r.verdicts.Delete(ctx, key)
r.ch.VerdictDeleted(ctx, key) // the watch of the verdicts hands the delete over
r.ch.Wait()
}
v := r.verdict(t, key)
if row.want == delivery.VerdictPass {
if v.State != delivery.VerdictPass || j.count() != row.judged {
t.Errorf("verdict %s after %d judgings, want a pass after %d", v.State, j.count(), row.judged)
}
return
}
if !IsUntrunked(v) || !strings.Contains(v.Failed.Words, row.want) || j.count() != 0 {
t.Errorf("verdict %+v after %d judgings, want a failure naming %s and none judged", v, j.count(), row.want)
}
for _, p := range row.pulls {
if s := owedState(t, r, p.Repository, p.Number, p.Head.Head); s != "error" {
t.Errorf("%s#%d owes %q, want error", p.Repository, p.Number, s)
}
}
})
}
}
func countWith(lines []string, words string) int {
n := 0
for _, l := range lines {
if strings.Contains(l, words) {
n++
}
}
return n
}
// 1887/A10, ADR 0527 §2: hq is judged as any repository, its checks Go tests, and its pull request gets a verdict and a
@@ -446,10 +333,7 @@ func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T
}
}
// No check of heads no candidate holds is left running (issue 1890): at start the sweep ends a key left running; while the
// checker runs, a check whose group moved to other heads is ended at once, unjudged, and writes no status; heads held again
// are checked whole; and a group's check starts only once its heads settled, so a partial set of members is never checked.
func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) {
func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T) {
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
@@ -467,91 +351,12 @@ func TestHeadsNoCandidateHoldsAreNeverLeftRunning(t *testing.T) {
if err := r.ch.Sweep(ctx); err != nil {
t.Fatal(err)
}
if v := r.verdict(t, dead); !IsEnded(v) || !strings.Contains(v.Failed.Words, "closed at start") {
if v := r.verdict(t, dead); v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "closed at start") {
t.Errorf("the dead key = %+v", v)
}
if v := r.verdict(t, live); v.State != delivery.VerdictRunning {
t.Errorf("the live key = %+v, want it left for the check to take over", v)
}
// A member joins while the check of the lone member's heads runs: that check is ended, its judge's run cancelled.
started, ended := make(chan struct{}), make(chan error, 1)
j = &judgeFunc{f: func(run context.Context, _ *Case, call int) (Outcome, error) {
if call == 1 {
close(started)
<-run.Done()
ended <- context.Cause(run)
return Outcome{}, run.Err()
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
var logged []string
var logMu sync.Mutex
r.ch.Logf = func(f string, a ...any) { logMu.Lock(); logged = append(logged, fmt.Sprintf(f, a...)); logMu.Unlock() }
alone, both := cand, candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 3, head2))
r.ch.Candidate(ctx, "feat%2Fx", alone)
<-started
r.ch.Candidate(ctx, "feat%2Fx", both)
var bothC delivery.Candidate
_ = json.Unmarshal(both, &bothC)
select {
case err := <-ended:
if err == nil || !strings.Contains(err.Error(), "superseded by "+GateHeads(bothC)) {
t.Errorf("the old check's run ended with %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("the check of the heads the candidate left still runs")
}
r.ch.Wait()
if v := r.verdict(t, live); !IsEnded(v) || v.Failed.Words != EndedWords+"superseded by "+GateHeads(bothC) {
t.Errorf("the superseded verdict = %+v", v.Failed)
}
if v := r.verdict(t, keyOf(t, "feat/x", both)); v.State != delivery.VerdictPass {
t.Errorf("the group's verdict = %s", v.State)
}
okey, _ := delivery.CheckerOwedKey(repoA, 7, head1)
if raw, _, _, _ := r.owed.Get(ctx, okey); !strings.Contains(string(raw), `"success"`) {
t.Errorf("the head's status = %s, want the group's success, never the ended check's", raw)
}
logMu.Lock()
said := 0
for _, l := range logged {
if strings.Contains(l, "ended: superseded") {
said++
}
}
logMu.Unlock()
if said != 1 {
t.Errorf("the end was said %d times: %v", said, logged)
}
// The member leaves again: the ended heads are held again and checked whole.
r.ch.Candidate(ctx, "feat%2Fx", alone)
r.ch.Wait()
if v := r.verdict(t, live); v.State != delivery.VerdictPass || j.count() != 3 {
t.Errorf("the heads held again = %s after %d judgings, want a pass of a third", v.State, j.count())
}
// Members announced one after another: only the settled, whole set is checked.
j = passJudge()
r = newRig(t, j)
r.ch.Settle = time.Hour
sdk := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2))
whole := candidateOf(t, "feat/y", pullOf(repoSDK, 4, head2), pullOf(repoA, 8, head1))
r.ch.Candidate(ctx, "feat%2Fy", sdk)
r.ch.Candidate(ctx, "feat%2Fy", whole)
r.ch.Reconsider(ctx) // a trunk move or a turn while the heads settle starts nothing
if j.count() != 0 {
t.Fatalf("judged %d times before the heads settled", j.count())
}
r.ch.settleNow()
r.ch.Wait()
if _, _, found, _ := r.verdicts.Get(ctx, keyOf(t, "feat/y", sdk)); found || j.count() != 1 {
t.Errorf("the partial set has a verdict (%v) or the judge ran %d times, want the whole set alone", found, j.count())
}
if v := r.verdict(t, keyOf(t, "feat/y", whole)); v.State != delivery.VerdictPass {
t.Errorf("the whole set = %s", v.State)
}
}
func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683 (the checker's half of AC5)
@@ -573,43 +378,3 @@ func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683
t.Errorf("told %v after a check that changed nothing", told)
}
}
// Sweep 2026-10-30 item 7: a pull request of a repository the mesh does not deliver (an operator's own, on the same
// forge) touches no module and its trunk the gate never names. It is outside the mesh: no verdict, no status owed (one
// owed before is taken away, so the forge module stops writing it into a repository the mesh may not write), said once.
func TestACandidateOutsideTheMeshIsJudgedNothingAndOwesNothing(t *testing.T) {
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
var logged []string
r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
const theirs = "someone/own-app"
okey, _ := delivery.CheckerOwedKey(theirs, 55, head1)
before, _ := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "error", Text: "cannot be judged"})
if _, err := r.owed.Create(ctx, okey, before); err != nil {
t.Fatal(err)
}
cand := candidateOf(t, "feat/theirs", pullOf(theirs, 55, head1))
key := keyOf(t, "feat/theirs", cand)
r.ch.Candidate(ctx, "feat/theirs", cand)
r.ch.Wait()
r.ch.Reconsider(ctx)
r.ch.Wait()
if _, _, found, _ := r.verdicts.Get(ctx, key); found || j.count() != 0 {
t.Errorf("a candidate outside the mesh was judged (%d) or given a verdict (%v)", j.count(), found)
}
if s := owedState(t, r, theirs, 55, head1); s != "" {
t.Errorf("%s#55 still owes %q on the forge", theirs, s)
}
if n := countWith(logged, "outside the mesh"); n != 1 {
t.Errorf("said %d times in the log, want once: %q", n, logged)
}
ckey, _ := delivery.CandidatesKey("feat/theirs")
st, err := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}.Status()
if err != nil {
t.Fatal(err)
}
if st.Outside["feat/theirs"] == "" || len(st.CannotBeJudged) != 0 || len(st.Unjudged) != 0 {
t.Errorf("status: outside %v, cannot be judged %v, unjudged %v", st.Outside, st.CannotBeJudged, st.Unjudged)
}
}
@@ -28,7 +28,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
b.ex.Compositions = f
cs := caseOf(t, pullOf(repoA, 7, head1))
cs.FPSet = "fps-of-the-candidate"
v := b.judge(t, cs).Verdict
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
@@ -39,15 +38,6 @@ func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
if !compositionMentions(v, "passed") {
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
}
if v.Composition.FPSet == "" || v.Composition.FPSet != cs.FPSet {
t.Errorf("the composition names the fingerprint set %q, want the candidate's %q (ADR 0548 §5)", v.Composition.FPSet, cs.FPSet)
}
// ADR 0548 §5: a carry whose fingerprints did not change keeps the composition it passed and asks the assigner nothing.
carry := caseOf(t, pullOf(repoA, 7, head1))
carry.CarriedFrom, carry.Composition = map[string]string{repoA: trunkA2}, v.Composition
if kept := b.judge(t, carry).Verdict; kept.Composition == nil || kept.Composition.SDK != "v0.1.14-x" || len(f.asked) != 1 {
t.Errorf("the carry's composition is %+v and the assigner was asked %d times; want the kept one and no second ask", kept.Composition, len(f.asked))
}
}
func compositionMentions(v delivery.Verdict, word string) bool {
@@ -5,7 +5,6 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
@@ -17,7 +16,6 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
@@ -118,9 +116,6 @@ type bench struct {
fetch Fetched
digest map[string]Digested // by level
test func(c stepCall) (any, error)
// declines is how many asks naming needs a node lacking them declines before shanks, which has every need, takes
// one (issue 890); an ask naming none runs on shanks, which then has none.
declines int
}
// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
@@ -156,17 +151,8 @@ func (b *bench) answer(a delivery.StepAsk) delivery.Step {
if module == buildrun.GoBuildModule {
steps.Digest = tool("digest")
}
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
if len(a.Needs) > 0 {
if b.declines > 0 {
b.declines--
result, _ := json.Marshal(delivery.StepDeclined{Declined: true, Needs: a.Needs, Lacks: a.Needs})
return delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: "ace", State: delivery.StepRefused, Started: began, Ended: began,
Reason: "declined", Result: result}
}
steps.Has = func(string) bool { return true }
}
out, err := buildstep.RunAsk(context.Background(), steps, a)
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
if err != nil {
step.State, step.Reason = delivery.StepFailed, err.Error()
@@ -274,36 +260,8 @@ func TestACleanMergeIsFetchedAtItsTrunkAndBothLevelsPass(t *testing.T) { // 1679
t.Errorf("%s was not kept", k)
}
}
// ADR 0567 §12: every check ask carries every node's name for the names check; a check whose names cannot be read, or
// that reads none, is not asked and fails saying so.
for _, c := range []struct {
names []string
err error
fails string
}{{[]string{"anchor", "laptop"}, nil, ""}, {nil, nil, "no node record"}, {nil, errors.New("no bucket"), "no bucket"}} {
b := newBench(t)
b.ex.Nodes = namesOf{c.names, c.err}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
asks := b.asks.Asks()
if c.fails != "" {
if out.Verdict.State != delivery.VerdictFail || len(asks) != 0 || !strings.Contains(out.Verdict.Failed.Words, c.fails) {
t.Errorf("names %v, %v: verdict %+v, %d asks", c.names, c.err, out.Verdict, len(asks))
}
} else if len(asks) == 0 || strings.Join(asks[0].Check.NodeNames, ",") != "anchor,laptop" {
t.Errorf("the check ask carries the names %v", asks)
}
}
}
// namesOf is the nodes' names as a test gives them.
type namesOf struct {
names []string
err error
}
func (n namesOf) Names(context.Context) ([]string, error) { return n.names, n.err }
func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
b := newBench(t)
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/slow": digA, "fixture/other": digC}}
@@ -318,49 +276,6 @@ func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
}
// Issue 890/AC5: a package needing node-build-node is left unmet where the node lacks it; the check keeps the passes
// and asks the integration level again for a node with it, puts it again when a node declines it, and that node tests
// the package alone, its kept passes not counted twice.
// fixture/outside has no kept verdict after its pass (it reads outside itself: its digest moves every call), so a
// placed ask that ran more than its unmet packages would run and count it again.
b = newBench(t)
b.declines = 1
outside := 0
b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
d := echoBeside(b, b.digest[c.args["level"].(string)])
if d.Level == "integration" {
outside++
d.Packages = map[string]string{"fixture/slow": digB, "fixture/node": digC, "fixture/outside": fmt.Sprintf("%064x", outside)}
}
return d, nil
}
b.digest[delivery.LevelIntegration] = Digested{Level: "integration",
Needs: []delivery.PackageNeeds{{Package: "fixture/node", Needs: []string{delivery.NeedNodeBuildNode}}}}
out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
asks := b.asks.Asks()
if out.Verdict.State != delivery.VerdictPass || len(asks) != 3 || len(asks[0].Needs) != 0 {
t.Fatalf("verdict %+v, asks %+v: want a pass of three asks, the first naming no need", out.Verdict, asks)
}
for _, a := range asks[1:] {
if strings.Join(a.Needs, ",") != delivery.NeedNodeBuildNode || strings.Join(a.Check.Levels, ",") != delivery.LevelIntegration || strings.Join(a.Check.Packages, ",") != "fixture/node" {
t.Errorf("a placed ask %+v, want the integration level needing node-build-node", a)
}
}
tests = b.steps.tool("test")
if last := tests[len(tests)-1]; len(tests) != 3 || strings.Join(packagesOf(last), ",") != "fixture/node" || last.args["level"] != "integration" {
t.Errorf("test calls = %+v, want the unit level, the integration level of fixture/slow and fixture/outside, then fixture/node alone", tests)
}
slow := captured[Tested](t, "go-test-slow")
if i := out.Verdict.Levels[delivery.LevelIntegration]; i.Run != 2*slow.Tests || i.Kept != 0 {
t.Errorf("integration = %+v, want %d run and nothing counted kept", i, 2*slow.Tests)
}
// And where no node has the need, the check fails once the declines outlast the bound, naming the need.
b.kept, b.declines = newMemKV(), 1000
b.ex.Kept, b.ex.DeclineBound = Kept{Store: b.kept}, 200*time.Millisecond
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict; v.State != delivery.VerdictFail || !strings.Contains(fmt.Sprint(v), "needs a node with node-build-node") {
t.Errorf("with no node having the need: %+v", v)
}
}
func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
@@ -426,9 +341,9 @@ func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1
}
}
// 1289/AC1, AC2 (R4c, mesh-sdk buildrun at 0b12b62): the change's own tests run first, and when one fails the level ends
// there: no other package of the level and no later level runs, so the rest of the suite is not paid for.
func TestTheChangesOwnFailingTestEndsTheLevelThere(t *testing.T) {
// 1289/AC1, AC2 as issue 1921 changed it: the change's own failing test runs first, the level's other packages still run
// after it (so one check names every failure), and no later level runs.
func TestTheChangesOwnFailingTestRunsFirstThenTheRestAndNoLaterLevel(t *testing.T) {
b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
@@ -444,8 +359,8 @@ func TestTheChangesOwnFailingTestEndsTheLevelThere(t *testing.T) {
t.Fatalf("verdict = %+v", out.Verdict)
}
tests := b.steps.tool("test")
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
t.Fatalf("test calls = %+v, want m/internal/b alone: its failure ends the level", tests)
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" || strings.Join(packagesOf(tests[1]), ",") != "m/internal/a" {
t.Fatalf("test calls = %+v, want m/internal/b, then m/internal/a, each once", tests)
}
if n := len(b.steps.tool("digest")); n != 1 {
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
@@ -705,8 +620,9 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversT
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
func echoBeside(b *bench, d Digested) Digested {
var lines []string
if fetches := b.steps.tool("fetch"); len(fetches) > 0 { // the workspace digested is the last fetch's (issue 890: a placed ask fetches again)
for _, r := range refsOf(fetches[len(fetches)-1].args["beside"]) {
for _, f := range b.steps.tool("fetch") {
refs, _ := refsOf(f.args["beside"]), true
for _, r := range refs {
lines = append(lines, r.Repository+" "+r.Commit)
}
}
@@ -929,81 +845,3 @@ func TestASecondCheckOfTheSameHeadsKeepsWhatTheFirstPassed(t *testing.T) {
t.Errorf("%d test steps ran on the second check", n)
}
}
// keptList is the indexer's kept predefined list as a test gives it.
type keptList string
func (k keptList) PredefinedList(context.Context) (json.RawMessage, error) {
return json.RawMessage(k), nil
}
// fixedDigests are the indexer's manifest digests as a test gives them.
type fixedDigests map[string]string
func (f fixedDigests) ManifestDigests() map[string]string { return f }
// ADR 0578 §2: a change's manifests are judged by the indexer's judge, built at mesh-delivery's trunk and run as the
// composer is (a compose ask of its command); a refusal fails the check naming the manifest and each problem; a group
// whose mesh-delivery member changes the judge is judged by that member's head.
func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
web, trunk := strings.Repeat("e", 64), strings.Repeat("d", 40)
for _, row := range []struct {
name string
refused string
fails string
// untouched is the judge's refusal of mail, which the change leaves as it is, and trunkMail its refusal on the trunk:
// a problem the trunk had is a warning, one the change causes fails (ADR 0579 §2)
untouched, trunkMail string
}{
{name: "accepted"},
{name: "an untouched manifest's problem the trunk had", untouched: "mail says no credential form", trunkMail: "mail says no credential form"},
{name: "a problem the change causes in an untouched manifest", untouched: "mail reads a state web no longer keeps",
fails: "refuses the manifest of mail: mail reads a state web no longer keeps"},
{name: "refused", refused: "web requires \"amqp\"\nweb has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web requires "amqp"; web has no id`},
} {
b := newBench(t)
b.ex.Manifests = fixedDigests{"mail": strings.Repeat("a", 64), "web": strings.Repeat("b", 64)}
b.ex.Trunks = fixedTrunks{heads: map[string]string{DeliveryRepository: trunk}}
b.ex.Predefined = keptList(`{"font-size": {"type": "number", "description": "the size"}}`)
var judged []delivery.ComposeJob
answer := b.asks.Answer
b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
if a.Compose == nil {
return answer(a)
}
judged = append(judged, *a.Compose)
read := composer.ManifestRead{Accepted: row.refused == "", Refused: row.refused}
verdict, _ := json.Marshal(composer.Verdict{V: composer.Version, SDK: "exe:1", Cases: []composer.CaseVerdict{{Name: JudgeCase,
Manifests: map[string]composer.ManifestRead{"web": read, "mail": {Accepted: row.untouched == "", Refused: row.untouched}}},
{Name: JudgeTrunk, Manifests: map[string]composer.ManifestRead{"web": {Accepted: true}, "mail": {Accepted: row.trunkMail == "", Refused: row.trunkMail}}}}})
result, _ := json.Marshal(delivery.ComposeAnswer{Key: "k", Verdict: verdict})
return delivery.Step{ID: a.ID, State: delivery.StepPassed, Result: result}
}
p := pullOf(repoA, 7, head1)
p.Modules = []string{"web"}
p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}}
v := b.judge(t, caseOf(t, p)).Verdict
if len(judged) != 1 || judged[0].Fetch["repository"] != DeliveryRepository || judged[0].Fetch["commit"] != trunk ||
judged[0].Package != JudgePackage || judged[0].Dir != JudgeDir {
t.Fatalf("%s: judged %+v (verdict %+v)", row.name, judged, v)
}
var req composer.Request
if err := json.Unmarshal(judged[0].Request, &req); err != nil || req.Cases[0].Manifests["web"].Digest != web ||
req.Cases[1].Name != JudgeTrunk || req.Cases[1].Manifests["web"].Digest != strings.Repeat("b", 64) ||
req.Predefined["font-size"].Type != "number" ||
req.Cases[0].Manifests["mail"].Digest != strings.Repeat("a", 64) {
t.Fatalf("%s: the request %+v: %v", row.name, req, err)
}
switch {
case row.fails == "" && v.State != delivery.VerdictPass:
t.Errorf("%s: %+v", row.name, v)
case row.fails != "" && (v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, row.fails)):
t.Errorf("%s: %+v, want a fail saying %q", row.name, v.Failed, row.fails)
}
}
if !changesJudge(DeliveryRepository, []Fetched{{Changed: []string{JudgeDir + "/judge/judge.go"}}}) ||
changesJudge(DeliveryRepository, []Fetched{{Changed: []string{JudgeDir + "/internal/index/x.go"}}}) ||
changesJudge("novox/mesh-catalog", []Fetched{{Changed: []string{JudgeDir + "/judge/judge.go"}}}) {
t.Error("which merge changes the judge")
}
}
@@ -42,7 +42,7 @@ func TestTheStatusLineNamesTheFailureNotTheFirstLoggedLine(t *testing.T) { // 19
f := fatalAfterLog()
f.Cause = ""
v = levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
if !strings.HasPrefix(afterHid(v.Failed.Words), "agreement_catalogue_test.go:130") {
if !strings.HasPrefix(v.Failed.Words, "agreement_catalogue_test.go:130") {
t.Errorf("words = %q", v.Failed.Words)
}
}
@@ -56,45 +56,18 @@ func TestWordsLongerThanTheBoundKeepTheirLastLines(t *testing.T) { // 1901/AC1
_ = delivery.VerdictFail
}
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before
// (1901/AC3). A failure that hides its cause (none, or a passing line) is said loudly first in the words and in the status
// line, naming the build module and the version it says it is, and the level still fails; a failure that keeps the
// contract is worded as before (1901/AC8).
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3, AC8
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before.
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3
old := `{"package":"p","test":"TestX","output":"x_test.go:1: logged\nx_test.go:2: the old kept lines"}`
var f Failure
if err := json.Unmarshal([]byte(old), &f); err != nil || f.Cause != "" {
t.Fatalf("1901/AC3: err = %v, cause = %q", err, f.Cause)
}
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
if afterHid(v.Failed.Words) != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
if v.Failed.Words != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
t.Errorf("1901/AC3: words = %q", v.Failed.Words)
}
if got := firstFailed(*v); !strings.Contains(got, "the build module hid the failure") || strings.Contains(got, "\n") {
t.Errorf("1901/AC3, AC8: status = %q", got)
}
by := "mesh-build-node with mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78"
for _, c := range []struct {
name, module, by, cause, output, loud string
}{
{"no cause", "mesh-build-node", by, "", "test/b.test.ts: something", "the build module hid the failure (mesh-build-node at mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78): it sent no cause"},
{"a passing line as its cause", "mesh-build-node", by, "\u2714 the same merge has the same id (8.006913ms)", "", "the build module hid the failure (mesh-build-node at mesh-sdk v0.1.14-0.20261026184535-e40dd0f56a78): its cause is a passing line"},
{"from before the contract", "mesh-build-node", "", "", "\u2714 the same merge has the same id (8.006913ms)", "the build module hid the failure (mesh-build-node at a version before the failure contract): it sent no cause"},
{"a real cause", "mesh-build-node", by, "one plus one is not three", "not ok 3 - fails", ""},
} {
t.Run(c.name, func(t *testing.T) {
v := levelFailure(&target{module: c.module}, "unit", Tested{By: c.by, Failed: []Failure{{Package: "modules/gitea", Test: "fails", Cause: c.cause, Output: c.output}}})
if v.State != delivery.VerdictFail {
t.Errorf("1901/AC8: the verdict is %q, want it failed", v.State)
}
line := firstFailed(*v)
switch {
case c.loud == "" && strings.Contains(v.Failed.Words, "hid the failure"):
t.Errorf("1901/AC8: a failure that keeps the contract is said hidden: %q", v.Failed.Words)
case c.loud != "" && (!strings.HasPrefix(v.Failed.Words, c.loud) || !strings.Contains(line, c.loud) || !strings.Contains(v.Failures[0].Words, "hid the failure")):
t.Errorf("1901/AC8: words %q, status %q, want both to start with %q", v.Failed.Words, line, c.loud)
}
})
if got := firstFailed(*v); !strings.Contains(got, "logged") || strings.Contains(got, "\n") {
t.Errorf("1901/AC3: status = %q", got)
}
}
@@ -60,11 +60,3 @@ func (m *memKV) Keys(context.Context) ([]string, error) {
sort.Strings(keys)
return keys, nil
}
func (m *memKV) Delete(_ context.Context, key string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.vals, key)
delete(m.revs, key)
return nil
}
@@ -4,6 +4,7 @@ import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -87,3 +88,29 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
t.Errorf("mesh-sdk#13, alone into a trunk that did not move, is %q, want its success to stand", s)
}
}
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
ctx := context.Background()
var r *rig
release, judged := make(chan struct{}), make(chan struct{})
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
if call == 1 {
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
defer close(judged)
} else {
<-release // the check asked again for the new trunk waits in the stream meanwhile
}
return Outcome{Verdict: passing()}, nil
}}
r = newRig(t, j)
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
<-judged
for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
}
time.Sleep(20 * time.Millisecond)
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
}
close(release)
r.ch.Wait()
}
@@ -0,0 +1,21 @@
package check
import (
"fmt"
"testing"
"github.com/nats-io/nats.go"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Issue 1820 AC5: a read of the forge that nobody answers is the forge's failure, not a build node's. (AC1-AC4 moved a
// step to another node; since R4g an ask whose node stops is delivered again by its stream: ADR 0525 §6.)
func TestAPinReadThatNobodyAnswersIsTheForgesFailureNotABuildNodes(t *testing.T) { // 1820/AC5
b := groupBench(t, "cb54756c3865")
b.ex.Forge.(*fakeForge).err = fmt.Errorf("gitea.gitea_get_file: %w", nats.ErrNoResponders)
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
if v.State != delivery.VerdictFail || v.Failed.Machine != "forge" {
t.Errorf("verdict = %+v, want a failure at the forge: no build node was to blame", v)
}
}
@@ -80,9 +80,6 @@ var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buil
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
type Examiner struct {
Asks Asks
// DeclineBound is how long a placed ask may go on being declined, from its first decline, before the check fails
// saying no node with its need took it (issue 890, ADR 0552); DefaultDeclineBound when zero.
DeclineBound time.Duration
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
Toolchains ToolchainEntries
Kept Kept
@@ -91,14 +88,9 @@ type Examiner struct {
Modules BuildModules
// Runs says which commit the mesh runs of a module (the indexer's modules); nil knows none.
Runs interface{ CommitOf(module string) string }
// Forge reads the forge's files for the siblings a repository's tests read; nil reads none.
// Forge reads the go.mod files of the Go members and says whether a mesh-sdk commit is on its trunk (ADR 0484 clause 9);
// nil checks no pin.
Forge Forge
// Published names the latest published version of every mesh package a module of the mesh uses, by package, from the
// indexer's modules (novox/hq ADR 0573 §2): a check builds against those, or against a member's head for a package
// its group publishes (§4), whatever go.mod names. Nil names none, and go.mod's versions are built.
Published interface {
LatestPackages() delivery.PackageVersions
}
// Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing
// of the builds.
Builds BuildStates
@@ -111,23 +103,12 @@ type Examiner struct {
}
// Trunks are the trunk heads of every repository, for the siblings a repository's tests read.
Trunks interface{ Heads() map[string]string }
// Manifests are the indexer's manifest digests by module: the catalogue a change's manifests are judged in by the
// indexer's judge (manifests_judge.go, novox/hq ADR 0578 §2); nil judges no manifest.
Manifests TrunkManifests
// Predefined reads the catalogue's predefined keys the indexer keeps (manifests_judge.go); nil reads none.
Predefined PredefinedList
// Compositions answers the assigner's composition check of a check by the check's key (delivery.CompositionKey),
// waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil
// leaves the composition not run.
Compositions CompositionReader
// Nodes names every node with a record, read from each `<node>.node` of the node-engine's have (novox/hq ADR 0567
// §12): every check ask carries them (CheckJob.NodeNames) for the names check, so no repository lists them. A check
// whose names cannot be read, or that reads none, is not asked and fails saying so; nil hands no names.
Nodes interface {
Names(ctx context.Context) ([]string, error)
}
Now func() time.Time
Logf func(format string, args ...any)
Now func() time.Time
Logf func(format string, args ...any)
}
// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote
@@ -222,27 +203,15 @@ type state struct {
nodes map[string]bool
clone time.Duration
slow []string
// beside are the siblings read at their trunk's head, by repository (ADR 0548 §6).
beside map[string]string
// judgeHead is the group's mesh-delivery member head when its merge changes the judge (ADR 0578 §2); empty judges
// with mesh-delivery's trunk.
judgeHead string
// predefinedHead is the catalogue member's head when its merge changes predefined.json: the list is read there.
predefinedHead string
}
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
// be judged at all.
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{},
beside: map[string]string{}}
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
if v := st.judgeAll(ctx); v != nil {
return st.finish(ctx, *v), nil
}
// Every manifest the change touches, judged by the indexer's judge (ADR 0578 §2): a refusal fails the check.
if v := st.judgeManifests(ctx); v != nil {
return st.finish(ctx, *v), nil
}
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
}
@@ -261,13 +230,6 @@ func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
if st.c.Composition != nil && st.c.Try == nil {
// A carry whose candidate's fingerprints did not change keeps the composition it passed (ADR 0548 §5).
kept := *st.c.Composition
v.Composition = &kept
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
return
}
if st.j.Compositions == nil || st.c.Try != nil {
return
}
@@ -281,9 +243,6 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
return
}
withComposition(v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
}
}
// outcome fills the verdict's levels, times, machines and composition from what ran.
@@ -323,9 +282,6 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
}
}
out := Outcome{Verdict: v, Slowest: slow}
if len(st.beside) > 0 {
out.Beside = st.beside
}
for _, t := range st.targets {
if t.judged != "" {
if out.Judged == nil {
@@ -387,7 +343,6 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
if len(beside) > 0 {
args["beside"] = beside
t.beside = besideLines(beside)
besideAtTrunk(st.c, beside, st.beside)
}
if p.Repository == catalogueRepository {
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
@@ -402,7 +357,6 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
st.j.writePreview(ctx, p, merged)
}
st.judgedByMember(p, merged)
if p.Repository == SDKRepository && len(st.c.Pulls) > 1 && st.j.changesPackage(p, merged) {
groupPackage = map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number)}
}
@@ -410,35 +364,11 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
return nil
}
// DefaultDeclineBound is an Examiner's DeclineBound when it names none: longer than a check runs, so a node with the
// need that is busy is waited for, and short enough that a mesh where no node has it says so.
const DefaultDeclineBound = 30 * time.Minute
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, then
// each level's digest, kept verdicts and tests. When the answer leaves packages unmet,
// their tests needing what that node lacks (issue 890), the passes are kept and the levels from the first such one are
// asked again of a node that has it (placed).
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
return st.askNeeding(ctx, t, merged, st.levelsOf(), nil, nil)
}
// askNeeding is ask with the levels to run, what the node must have and the packages it tests: none for the first ask
// of a target; for the placed one the unmet packages' needs and those packages alone, so no package the first ask judged
// is run or counted again (ADR 0552 §4).
func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, levels, needs, packages []string) *delivery.Verdict {
p := t.pull
placed := len(needs) > 0
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
if st.j.Nodes != nil {
names, err := st.j.Nodes.Names(ctx)
if err == nil && len(names) == 0 {
err = fmt.Errorf("no node record is on the bus")
}
if err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d is not asked: the nodes' names the names check needs cannot be read: %v", p.Repository, p.Number, err))
}
job.NodeNames = names
}
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
if t.runner != "" {
job.Runner, job.RunnerWhen = t.runner, buildGoDir
}
@@ -450,56 +380,17 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Check: job, Needs: needs}
if a.Language == delivery.AskGo {
a.Packages = st.packagesOf(p.Repository)
At: st.j.now().UTC(), Check: job}
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
if err := st.j.Asks.Put(ctx, a); err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
}
var step delivery.Step
var firstDecline time.Time
for {
what := fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number)
if placed {
what += " for a node with " + strings.Join(needs, ", ")
step, err := st.j.Asks.Wait(ctx, a)
if err != nil {
if ctx.Err() != nil {
return fail(t.module, ctx.Err().Error())
}
st.c.Progress.Set(what, "")
st.c.Asked(a.ID)
if err := st.j.Asks.Put(ctx, a); err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
}
waiting := ctx
if watch := st.c.Watch; watch != nil {
id := a.ID
watch(id, nil) // put: a try names it at once, not only when it ends (issue 1890)
waiting = buildrun.WithStepSeen(ctx, func(s delivery.Step) { watch(id, &s) })
}
var err error
step, err = st.j.Asks.Wait(waiting, a)
if err != nil {
if ctx.Err() != nil {
return fail(t.module, ctx.Err().Error())
}
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
}
d, declined := delivery.DeclinedOf(step)
if !declined {
break
}
// A node that lacks the need dropped the ask (issue 890): it is put again, for a node that has it to take; the
// node that declined looks at the stream again only after its next pull's wait.
now := st.j.now()
if firstDecline.IsZero() {
firstDecline = now
}
bound := st.j.DeclineBound
if bound <= 0 {
bound = DefaultDeclineBound
}
if now.Sub(firstDecline) > bound {
return fail(step.Node, fmt.Sprintf("the check of %s#%d needs a node with %s, and none took it in %s: every node that pulled it lacks it (last %s)",
p.Repository, p.Number, strings.Join(needs, ", "), bound, step.Node))
}
st.j.logf("%s#%d: the ask %s was declined on %s, which lacks %s; asked again", p.Repository, p.Number, a.ID, step.Node, strings.Join(d.Lacks, ", "))
a.ID, a.At = buildstep.NewID(), now.UTC()
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
}
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
took := time.Duration(0)
@@ -510,7 +401,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
st.nodes[step.Node] = true
st.c.Progress.Set(fmt.Sprintf("judge the check of %s#%d", p.Repository, p.Number), step.Node)
if step.State != delivery.StepPassed {
return fail(step.Node, fmt.Sprintf("the check of %s#%d %s: %s", p.Repository, p.Number, step.State, step.Reason))
return fail(step.Node, fmt.Sprintf("the check of %s#%d %s: %s%s", p.Repository, p.Number, step.State, step.Reason, pinHint(st.c, step.Reason)))
}
var answer delivery.CheckAnswer
if err := json.Unmarshal(step.Result, &answer); err != nil {
@@ -523,28 +414,18 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
if st.c.Try == nil {
st.j.writePreview(ctx, p, append(*merged, t.fetched))
}
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
}
t.changed = t.fetched.Changed
if !changesBuildGo(p.Repository, t.changed) {
t.runner = "" // the build module used no runner: the merge does not change it
}
if !placed { // the placed ask fetched the same merge: its preview was read from the first
*merged = append(*merged, t.fetched)
if v := st.checkBinariesOf(t); v != nil {
return v
}
*merged = append(*merged, t.fetched)
if v := st.checkPinsOf(ctx, t); v != nil {
return v
}
ran := 0.0
for _, la := range answer.Levels {
if placed {
if len(la.Unmet) > 0 {
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
return fail(step.Node, fmt.Sprintf("the %s level of %s#%d left %s untested on %s, which was asked for a node with %s",
la.Level, p.Repository, p.Number, unmetNames(la.Unmet), step.Node, strings.Join(needs, ", ")))
}
la.Kept = nil // it names its packages and takes no other's kept verdict; this is only a guard against an older module
}
v, elapsed := st.level(ctx, t, la)
ran += elapsed
if v != nil {
@@ -553,48 +434,9 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
}
}
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
if unmet := delivery.UnmetNeeds(answer.Levels); len(unmet) > 0 && !placed {
// Every level passed, its passes kept, and some packages were left to a node with what their tests need: the levels
// from the first that left one are asked of such a node, which takes the kept passes and tests those (issue 890).
var again, packages []string
seen := map[string]bool{}
for _, la := range answer.Levels {
if len(again) > 0 || len(la.Unmet) > 0 {
again = append(again, la.Level)
}
for _, u := range la.Unmet {
if !seen[u.Package] {
seen[u.Package] = true
packages = append(packages, u.Package)
}
}
}
st.j.logf("%s#%d: %s on %s left untested; asked of a node with %s", p.Repository, p.Number, unmetOf(answer.Levels), step.Node, strings.Join(unmet, ", "))
return st.askNeeding(ctx, t, merged, again, unmet, packages)
}
return nil
}
// unmetNames names the unmet packages and their needs.
func unmetNames(unmet []delivery.PackageNeeds) string {
var out []string
for _, u := range unmet {
out = append(out, u.Package+" (needs "+strings.Join(u.Needs, ", ")+")")
}
return strings.Join(out, ", ")
}
// unmetOf names the unmet packages of every level.
func unmetOf(levels []delivery.LevelAnswer) string {
var out []string
for _, la := range levels {
if len(la.Unmet) > 0 {
out = append(out, "at "+la.Level+" "+unmetNames(la.Unmet))
}
}
return strings.Join(out, "; ")
}
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
// of the packages that passed, the kept ones taken are counted, and every test result is read and counted, also after
// one that did not pass, so the verdict names every failed test of the level (issue 1921). It answers the level's
@@ -699,12 +541,6 @@ const MaxWords = 4096
// no cause told apart (an older build module) the cause is the output's first "panic:", "fatal error:" or "signal:"
// line, read before the output is cut, or the words say the tests passed and the program failed after them.
func failureWords(f Failure) string {
return failureWordsAfter(f, "")
}
// failureWordsAfter is failureWords with the line loud put first, within the same MaxWords: what the checker itself says of
// the failure (that the build module hid it), which the status line then names.
func failureWordsAfter(f Failure, loud string) string {
out, passed := dropSummary(strings.TrimSpace(f.Output))
cause := strings.TrimSpace(f.Cause)
if cause == "" {
@@ -720,14 +556,11 @@ func failureWordsAfter(f Failure, loud string) string {
if len(cause) > MaxCause {
cause = strings.ToValidUTF8(cause[:MaxCause], "") + "..."
}
// The words keep MaxWords exactly: the loud line, the cause and their newlines, the marker and the kept lines.
// The words keep MaxWords exactly: the cause and its newline, the marker and the kept lines (review round 1).
budget := MaxWords
if cause != "" {
budget -= len(cause) + 1
}
if loud != "" {
budget -= len(loud) + 1
}
if len(out) > budget {
const marker = "... "
tail := out[len(out)-(budget-len(marker)):]
@@ -740,31 +573,9 @@ func failureWordsAfter(f Failure, loud string) string {
if cause != "" && !strings.HasPrefix(firstLine(out), cause) {
out = strings.TrimRight(cause+"\n"+out, "\n")
}
if loud != "" {
out = strings.TrimRight(loud+"\n"+out, "\n")
}
return out
}
// hidWords is the checker's loud line for a failure the build module sent hiding its cause (issue 1901, its thorough
// fix): with no cause, or a passing line as its cause, under the failure contract (buildstep.Hidden). It names the build
// module and the version it says it is (Tested.By, "<module> with mesh-sdk <version>"); one from before the contract
// says nothing of itself. "" when the failure keeps the contract. The check fails either way: only its words change.
func hidWords(module, by string, f Failure) string {
why := buildstep.Hidden(f)
if why == "" {
return ""
}
version := "a version before the failure contract"
if module == "" {
module = "an unnamed build module"
}
if m, v, ok := strings.Cut(by, " with "); ok && m != "" {
module, version = m, v
}
return fmt.Sprintf("the build module hid the failure (%s at %s): %s", module, version, why)
}
// MaxCause is the most a cause in a failure's words holds: one long line (the 1901 review: a cause of a kilobyte could
// push the words past MaxWords).
const MaxCause = 1024
@@ -851,7 +662,7 @@ func levelFailure(t *target, level string, results ...Tested) *delivery.Verdict
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
}
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWordsAfter(f, hidWords(t.module, res.By, f))})
all = append(all, delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)})
}
}
if len(all) == 0 { // no result: a level said failed with nothing to name
@@ -56,22 +56,12 @@ func (k Kept) Keep(ctx context.Context, level, digest string, v delivery.Kept) (
// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module
// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause
// 1): it runs every check. A package with skipped tests is not kept either: a pass that skipped (a test needing a
// toolchain the node lacks) would be reused for its digest and never run anywhere (issue 890). A build module that
// counts skips only per result, not per package, keeps nothing of a result in which a test skipped. It answers the
// packages it kept now.
// 1): it runs every check. It answers the packages it kept now.
func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) {
var kept []string
perPackage := false
for _, p := range res.Packages {
perPackage = perPackage || p.Skipped > 0
}
if res.Skipped > 0 && !perPackage {
return kept, nil // an older build module: the skip cannot be placed in a package
}
for _, p := range res.Packages {
digest, ok := digests[p.Package]
if !ok || !p.Passed || p.NoTests || p.Tests == 0 || p.Skipped > 0 {
if !ok || !p.Passed || p.NoTests || p.Tests == 0 {
continue
}
if failedIn(res, p.Package) {
@@ -74,32 +74,6 @@ func TestOnlyPassingPackagesWithADigestAreKept(t *testing.T) { // 1287/AC3, AC4
}
}
func TestAResultWithSkippedTestsKeepsNothing(t *testing.T) { // issue 890: a pass that skipped proved nothing
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{{Package: "m/a", Passed: true, Tests: 4}}}
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA}, at0)
if err != nil || len(kept) != 0 {
t.Fatalf("kept %v, %v, want nothing", kept, err)
}
if keys, _ := st.Keys(ctx); len(keys) != 0 {
t.Errorf("the bucket holds %v, want none", keys)
}
}
func TestOnlyTheSkippingPackageOfAResultIsWithheld(t *testing.T) { // issue 890
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
res := Tested{Level: "integration", Passed: true, Skipped: 2, Packages: []GoPackage{
{Package: "m/a", Passed: true, Tests: 4, Skipped: 2},
{Package: "m/b", Passed: true, Tests: 3},
}}
kept, err := k.KeepPasses(ctx, "novox/mesh-catalog", "integration", "go1.26.0", res, map[string]string{"m/a": digA, "m/b": digB}, at0)
if err != nil || strings.Join(kept, ",") != "m/b" {
t.Fatalf("kept %v, %v, want only m/b", kept, err)
}
}
func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
@@ -85,22 +85,18 @@ type pieces struct {
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
judged string
// carried says, of a carried verdict, the commits it was carried from (ADR 0548 §8); never dropped.
carried string
// warning says that a part of the check did not run, for instance that an older build module could not refuse a compiled program; never dropped.
warning string
}
// pullKey names a pull request of a case: `<owner/repo>#<number>`.
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
func (p pieces) join(drop int) string {
parts := []string{p.head, p.commits, p.carried, p.warning, p.counts, p.judged}
parts := []string{p.head, p.commits, p.counts, p.judged}
switch {
case drop < 3 && p.times != "":
parts = append(parts, p.times)
case drop == 3 && p.shortTimes != "":
parts = []string{p.head, p.commits, p.carried, p.warning, p.shortCounts, p.judged, p.shortTimes}
parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
}
if p.over != "" {
parts = append(parts, p.over)
@@ -175,62 +171,16 @@ func buildWords(builds []BuildNote) string {
return "builds: " + strings.Join(parts, ", ")
}
// carriedWords say the commits a verdict was carried from, for the line of pull request p (ADR 0548 §8): its own trunk's
// commit as twelve characters, a sibling's with its name before it; only those that moved, sorted. Empty when none did.
func carriedWords(from map[string]string, v delivery.Verdict, cs *Case, p Pull) string {
var own string
var others []string
for r, old := range from {
now, ok := v.Trunk[r]
if cs != nil && !ok {
now, ok = cs.Trunk[r]
}
if !ok {
now = v.Beside[r]
}
if now == "" || now == old {
continue
}
if strings.EqualFold(r, p.Repository) {
own = short12(old)
continue
}
name := r
if i := strings.LastIndex(r, "/"); i >= 0 {
name = r[i+1:]
}
others = append(others, name+" "+short12(old))
}
slices.Sort(others)
if own != "" {
others = append([]string{own}, others...)
}
return strings.Join(others, ", ")
}
// parts makes the pieces of the line of a pull request of a case.
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
var pc pieces
pc.judged = judged[pullKey(p)]
if cs != nil {
pc.warning = cs.Warning(pullKey(p))
}
commits := short12(p.Head) + " on " + short12(p.Trunk)
switch v.State {
case delivery.VerdictRunning:
pc.head = "checking " + commits
if cs != nil && cs.CarriedFrom != nil {
// Out of date and being carried to the new commit (ADR 0548 §6): pending, and saying so.
pc.head = "being carried to " + commits
if w := carriedWords(cs.CarriedFrom, v, cs, p); w != "" {
pc.carried = "from " + w
}
}
case delivery.VerdictPass:
pc.head = "pass " + commits
if w := carriedWords(v.CarriedFrom, v, cs, p); w != "" {
pc.carried = "carried from " + w // the counts that follow say what the carry ran and kept
}
default:
pc.head = "failed: " + firstFailed(v)
if n := failedCount(v); n > 1 {
@@ -1,39 +0,0 @@
package check
import (
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **An ask the build seat gave up on ends its step** (novox/hq issue 734): mesh-sdk buildrun.LostAskWaiter hands the
// waiting asker a failed step on the node buildrun.LostNode when DEAD_LETTERS keeps the ask's copy (buildrun.HearDeadLetters,
// granted by buscompose DeadLetterReads), and, under buildrun.NotTakenBound, when no build node took the ask within the
// bound. What follows is the checker's: a try ends as error (Tryer.run), with the not-taken bound at the step bound
// (Tryer.NotTaken); a check is asked again under its claim, at most LostAskTries times, then fails as not heard and is
// judged again when the nodes change (Checker.try). A check's ask has no not-taken bound: it waits in the stream as long as
// the backlog is (ADR 0525 §9), and a check ended for a backlog would owe its pull request a failure.
// LostAskTries is how often a check asks again whose ask was given up, before it fails as not heard (UnheardWords) and is
// judged again when the nodes change.
const LostAskTries = 3
// IsLostAsk says whether a verdict failed because its ask judged nothing: the build seat gave it up (buildrun.LostWords), or
// the build module that ran its step was replaced under it, and its next process ended the step it left running
// (buildstep.RestartReason; a delivery of the build module, as on 2026-10-28). Neither is the change's failure: the check
// is asked again, as for a given-up ask.
func IsLostAsk(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && (strings.Contains(v.Failed.Words, buildrun.LostWords) ||
strings.Contains(v.Failed.Words, buildstep.RestartReason))
}
// IsNotTaken says whether a verdict failed because no build node took its ask within the bound (buildrun.NotTakenWords).
func IsNotTaken(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.NotTakenWords)
}
// HearingBucket is the checker's state hearing: whether it hears DEAD_LETTERS (buildrun.Hearing under
// buildrun.HearingKey), which checker_status says (dead_letters).
const HearingBucket = delivery.CheckerModule + "_" + buildrun.HearingState
@@ -1,347 +0,0 @@
package check
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// losingAsks is the build seat over a bus that gives up the first `lose` asks put (DEAD_LETTERS keeps them, as on
// 2026-10-26 during a bus reload: try-1793017340003993788) and answers the rest; with never set it neither answers nor
// gives up (an ask no node takes).
type losingAsks struct {
*buildrun.FakeAsks
answer func(delivery.StepAsk) delivery.Step
dead *buildrun.DeadLetters
never bool
mu sync.Mutex
lose int
}
func (l *losingAsks) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
if err := l.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
return err
}
l.mu.Lock()
lost := l.lose > 0
if lost {
l.lose--
}
l.mu.Unlock()
switch {
case l.never:
case lost:
h := map[string][]string{buildrun.DeadHeaderMsgID: {id}, buildrun.DeadHeaderDeliveries: {"5"}, buildrun.DeadHeaderStream: {"SEAT_NODE_BUILD_GO"},
buildrun.DeadHeaderGaveUp: {"2026-10-26T12:26:40Z"}}
g, ok := buildrun.GivenUpOf(h)
if !ok {
return nil
}
go l.dead.Heard(g)
default:
a, err := delivery.DecodeStepAsk(data)
if err != nil {
return err
}
l.Write(l.answer(a))
}
return nil
}
func losing(b *bench, lose int) *losingAsks {
l := &losingAsks{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, dead: &buildrun.DeadLetters{}, lose: lose}
b.ex.Asks = buildrun.Asker{Publisher: l, Waiter: buildrun.LostAskWaiter{Inner: l.FakeAsks, Dead: l.dead}, Bound: 5 * time.Second}
return l
}
// endable is the tries state as the end tool sees it.
type endable struct{ kv *memKV }
func (e endable) Entry(key string) (json.RawMessage, uint64, error) {
v, rev, _, err := e.kv.Get(context.Background(), key)
return v, rev, err
}
func (e endable) Update(key string, value []byte, revision uint64) error {
_, err := e.kv.Update(context.Background(), key, value, revision)
return err
}
// Issue 734: a try whose ask the build seat gave up on ends as error saying so, and the next ask of its head is a new try;
// one no node takes ends at the step bound; one ended by hand frees its place and stops being waited for.
func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testing.T) {
ctx := context.Background()
for _, row := range []struct {
name string
lose int
never bool
want string
}{
{"given up by the build seat", 1, false, buildrun.LostWords + " after 5 deliveries at 2026-10-26T12:26:40Z"},
{"taken by no build node within the bound", 0, true, buildrun.NotTakenWords + " within 200ms"},
} {
t.Run(row.name, func(t *testing.T) {
kv := newMemKV()
at := time.Date(2026, 10, 26, 12, 22, 20, 0, time.UTC)
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at)
if err != nil {
t.Fatal(err)
}
b := newBench(t)
l := losing(b, row.lose)
l.never = row.never
tr := &Tryer{Tries: kv, Judge: b.ex, NotTaken: 200 * time.Millisecond}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
got := readTry(t, kv, id)
if got.State != TryError || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], row.want) || !strings.Contains(got.Said, row.want) {
t.Fatalf("try = %+v, want error saying %q", got, row.want)
}
again, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, at.Add(time.Minute))
if err != nil || again == id {
t.Errorf("the head asked again answered %q, %v, want a new try", again, err)
}
})
}
t.Run("ended by hand", func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
b := newBench(t)
losing(b, 0).never = true
tr := &Tryer{Tries: kv, Judge: b.ex} // no bound: only the hand ends it
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
deadline := time.Now().Add(2 * time.Second)
for readTry(t, kv, id).State != TryRunning && time.Now().Before(deadline) {
time.Sleep(5 * time.Millisecond)
}
if _, err := EndTry(endable{kv}, id, "", time.Now()); err == nil {
t.Error("an end without a why was taken")
}
ended, err := EndTry(endable{kv}, id, "its ask was lost in the bus reload", time.Now())
if err != nil || ended.State != TryError || !strings.HasPrefix(ended.Said, "ended by hand ("+TryEndTool+"): its ask was lost") {
t.Fatalf("ended = %+v, %v", ended, err)
}
raw, _, _, _ = kv.Get(ctx, id)
tr.Put(ctx, id, raw) // the watch of the tries hands the end over
tr.Wait() // returns: the try's judging stopped
if got := readTry(t, kv, id); got.State != TryError || got.Said != ended.Said {
t.Errorf("try = %+v, want the hand's end kept", got)
}
if _, err := EndTry(endable{kv}, id, "again", time.Now()); err == nil || !strings.Contains(err.Error(), "ended already") {
t.Errorf("a second end answered %v", err)
}
})
}
// troubled is the build seat as a check meets it when its step's end does not reach its wait: the first `restarts` asks'
// steps are ended by the build module's next process (buildstep.RestartReason), and the first `deaf` asks' watches hear
// nothing though their steps end (the lost check of 2026-10-28). Its steps state is read through AskSteps and Steps.
type troubled struct {
*buildrun.FakeAsks
answer func(delivery.StepAsk) delivery.Step
mu sync.Mutex
restarts, deaf int
deafIDs map[string]bool
steps map[string]delivery.Step
}
func (tr *troubled) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
if err := tr.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
return err
}
a, err := delivery.DecodeStepAsk(data)
if err != nil {
return err
}
step := tr.answer(a)
step.What = buildstep.AskWhat(a)
tr.mu.Lock()
if tr.restarts > 0 {
tr.restarts--
step.State, step.Reason, step.Result = delivery.StepFailed, buildstep.RestartReason, nil
}
deaf := tr.deaf > 0
if deaf {
tr.deaf--
tr.deafIDs[id] = true
}
tr.steps[id] = step
tr.mu.Unlock()
if !deaf {
tr.Write(step)
}
return nil
}
func (tr *troubled) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
tr.mu.Lock()
deaf := tr.deafIDs[askID]
tr.mu.Unlock()
if deaf {
return make(chan delivery.Step), nil // open, and silent for good
}
return tr.FakeAsks.WatchAsk(ctx, module, askID)
}
func (tr *troubled) AskSteps(_ context.Context, _, askID string) ([]delivery.Step, error) {
tr.mu.Lock()
defer tr.mu.Unlock()
if s, ok := tr.steps[askID]; ok {
return []delivery.Step{s}, nil
}
return nil, nil
}
// Steps is the steps state as the read tools read it, with running in place of every step the hand has not let end.
func (tr *troubled) Steps(running bool) StateRead {
tr.mu.Lock()
defer tr.mu.Unlock()
vals := map[string]json.RawMessage{}
for id, s := range tr.steps {
if running {
s.State, s.Ended, s.Started = delivery.StepRunning, time.Time{}, time.Now()
}
raw, _ := delivery.EncodeStep(s)
vals[s.Node+"."+id] = raw
}
return StateRead{Keys: func() ([]string, error) {
var out []string
for k := range vals {
out = append(out, k)
}
return out, nil
}, Get: func(k string) (json.RawMessage, error) { return vals[k], nil }}
}
// kvRead is a bucket in memory as the read tools read it, live.
func kvRead(kv *memKV) StateRead {
return StateRead{Keys: func() ([]string, error) { return kv.Keys(context.Background()) },
Get: func(k string) (json.RawMessage, error) {
v, _, _, err := kv.Get(context.Background(), k)
return v, err
}}
}
// asVerdicts is the rig's verdicts as the ask-again tool reads and writes them.
type asVerdicts struct{ kv *memKV }
func (a asVerdicts) Entry(key string) (json.RawMessage, uint64, error) {
v, rev, _, err := a.kv.Get(context.Background(), key)
return v, rev, err
}
func (a asVerdicts) Update(key string, value []byte, revision uint64) error {
_, err := a.kv.Update(context.Background(), key, value, revision)
return err
}
// Issue 734 and the lost check of 2026-10-28: a check whose ask judged nothing is asked again, and one whose step ended
// unheard is judged. Given up by the build seat once, or its step ended by the build module's next process after a
// delivery replaced it, the check is asked again under its claim and passes; given up every time it fails loudly as not
// heard, judged again when the nodes change. A step whose watch went deaf is read at the next step end of its build module
// (a nudge) and judged with no new ask; one no event reaches is named lost by checker_status, refused by checker_ask_again
// while its step still runs, and asked again by it once its step ended.
func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
ctx := context.Background()
for _, row := range []struct {
name string
lose, restarts, deaf int
how string // "nudge": a step of the build module ends; "hand": checker_ask_again
state string
asks int
}{
{name: "given up once", lose: 1, state: delivery.VerdictPass, asks: 2},
{name: "given up every time", lose: LostAskTries, state: delivery.VerdictFail, asks: LostAskTries},
{name: "its build module replaced under it", restarts: 1, state: delivery.VerdictPass, asks: 2},
{name: "replaced under every ask", restarts: LostAskTries, state: delivery.VerdictFail, asks: LostAskTries},
{name: "its watch deaf, a step of its build module ends", deaf: 1, how: "nudge", state: delivery.VerdictPass, asks: 1},
{name: "its watch deaf, asked again by hand", deaf: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
{name: "its watch deaf, asked again by hand after its build module replaced it", deaf: 1, restarts: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
} {
t.Run(row.name, func(t *testing.T) {
b := newBench(t)
var asked func() int
var heard *buildrun.HeardEnds
var tr *troubled
if row.lose > 0 {
l := losing(b, row.lose)
asked = func() int { return len(l.Asks()) }
} else {
tr = &troubled{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, restarts: row.restarts, deaf: row.deaf,
deafIDs: map[string]bool{}, steps: map[string]delivery.Step{}}
heard = &buildrun.HeardEnds{Inner: tr, Read: tr}
b.ex.Asks = buildrun.Asker{Publisher: tr, Waiter: heard, Bound: 5 * time.Second}
asked = func() int { return len(tr.Asks()) }
}
r := newRig(t, b.ex)
asksKept := newMemKV()
r.ch.Asks = asksKept
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
key := keyOf(t, "feat/x", cand)
r.ch.Candidate(ctx, "feat%2Fx", cand)
if row.deaf > 0 {
for deadline := time.Now().Add(2 * time.Second); heard.Waiting() == 0 && time.Now().Before(deadline); {
time.Sleep(2 * time.Millisecond)
}
if v := r.verdict(t, key); v.State != delivery.VerdictRunning {
t.Fatalf("a check whose step ended unheard = %s, want running until it is read", v.State)
}
}
switch row.how {
case "nudge":
heard.Nudge(buildrun.NodeBuildModule) // another build module's step: not this one's to read
heard.Nudge(buildrun.GoBuildModule)
case "hand":
cands := newMemKV()
ckey, _ := delivery.CandidatesKey("feat/x")
_, _ = cands.Create(ctx, ckey, cand)
none := StateRead{Keys: func() ([]string, error) { return nil, nil }, Get: func(string) (json.RawMessage, error) { return nil, nil }}
rd := Reader{CandidateState: kvRead(cands), VerdictState: kvRead(r.verdicts), AskState: kvRead(asksKept),
StepStates: map[string]StateRead{buildrun.GoBuildModule: none}} // no node took its ask yet
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "waits in its build seat's stream") {
t.Errorf("asked again while its ask waits in the stream (issue 1791's kind): %v", err)
}
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(true)
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "a step of it runs") {
t.Errorf("asked again while its step runs: %v", err)
}
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(false)
if st, _ := rd.Status(); !strings.Contains(st.Lost["feat/x"], AskAgainTool) {
t.Errorf("status lost = %v, want feat/x named with %s", st.Lost, AskAgainTool)
}
got, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf")
if err != nil || got.Key != key || !strings.HasPrefix(got.Said, AskedAgainWords) {
t.Fatalf("asked again = %+v, %v", got, err)
}
raw, _, _, _ := r.verdicts.Get(ctx, key)
r.ch.VerdictWritten(ctx, key, raw) // the checker's watch of its verdicts hands it over
}
r.ch.Wait()
v := r.verdict(t, key)
if v.State != row.state || asked() != row.asks {
t.Errorf("verdict %s (%s) after %d asks, want %s after %d", v.State, v.Failed.Words, asked(), row.state, row.asks)
}
if _, _, kept, _ := asksKept.Get(ctx, key); kept {
t.Errorf("the asks of the ended check are still kept under %s", key)
}
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
t.Errorf("a check that judged nothing every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
}
if row.how == "hand" {
rd := Reader{VerdictState: kvRead(r.verdicts), CandidateState: kvRead(newMemKV())}
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "again"); err == nil {
t.Error("a judged check was asked again")
}
}
})
}
}
@@ -1,259 +0,0 @@
package check
import (
"context"
"encoding/json"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A change's manifests are judged by the indexer's judge (novox/hq ADR 0578 §2): the checker asks the Go build seat to
// build the judge's command (mesh-delivery's modules/mesh-delivery-indexer, cmd/mesh-judge) at mesh-delivery's trunk, or
// at the group's mesh-delivery member head when that member changes the judge, and to run it in its sandbox over the
// catalogue as the change leaves it, exactly as the composer is run (delivery.ComposeJob, ADR 0482 §4). A manifest the
// change touches that the judge refuses fails the check, naming the manifest and each problem. The checker links no
// judge of its own.
// The judge the checker runs.
const (
DeliveryRepository = "novox/mesh-delivery"
JudgeDir = "modules/mesh-delivery-indexer"
JudgePackage = "mesh-delivery-indexer/cmd/mesh-judge"
// JudgeCase is the case of a judge request holding the catalogue as the change leaves it, and JudgeTrunk the one
// holding it as the trunk holds it: a problem the change case has and the trunk case lacks is the change's (ADR 0579 §2).
JudgeCase = "change"
JudgeTrunk = "trunk"
// judgeBetween is how the judge names the problems between manifests that no one module's removal ends.
judgeBetween = "(between manifests)"
)
// judgePaths are the paths of mesh-delivery whose change changes the judge: a group whose mesh-delivery member changes
// one is judged by that member's head.
var judgePaths = []string{JudgeDir + "/judge/", JudgeDir + "/cmd/mesh-judge/", JudgeDir + "/go.mod", JudgeDir + "/go.sum"}
// changesJudge says whether a merge of mesh-delivery with these changed paths changes the judge.
func changesJudge(repository string, merged []Fetched) bool {
if repository != DeliveryRepository {
return false
}
for _, f := range merged {
for _, p := range f.Changed {
for _, j := range judgePaths {
if p == j || strings.HasPrefix(p, j) {
return true
}
}
}
}
return false
}
// TrunkManifests says the manifest digest of every module the indexer holds (its modules state), by module.
type TrunkManifests interface {
ManifestDigests() map[string]string
}
// PredefinedList is the catalogue's list of predefined keys the indexer keeps (its state predefined, key list), the one a
// config value falling back to a key is judged against (ADR 0348 §7); nil when it keeps none.
type PredefinedList interface {
PredefinedList(ctx context.Context) (json.RawMessage, error)
}
// judgeRequest is the judge's request over the catalogue as the case leaves it: every trunk manifest by its digest, each
// one a pull request touches replaced by the digest its candidate names; touched are the modules whose manifest the
// change adds or changes (a digest the trunk does not hold for it), sorted (novox/hq ADR 0579 §2).
func judgeRequest(trunk map[string]string, pulls []Pull) (req composer.Request, touched []string) {
manifests, before := map[string]composer.Manifest{}, map[string]composer.Manifest{}
for module, digest := range trunk {
if digest != "" {
manifests[module] = composer.Manifest{Digest: digest}
before[module] = composer.Manifest{Digest: digest}
}
}
seen := map[string]bool{}
for _, p := range pulls {
for module, cm := range p.Fps {
if cm.Manifest == "" {
continue
}
manifests[module] = composer.Manifest{Digest: cm.Manifest}
if trunk[module] != cm.Manifest {
seen[module] = true
}
}
}
return composer.Request{V: composer.Version, Cases: []composer.Case{{Name: JudgeCase, Manifests: manifests},
{Name: JudgeTrunk, Manifests: before}}}, slices.Sorted(maps.Keys(seen))
}
// judgeManifests runs the indexer's judge over the case's manifests and fails the check for each touched manifest it
// refuses. A case that touches no manifest, or an examiner that reads no trunk manifests, judges nothing; a judge that
// cannot be asked or does not answer fails the check, said, never passes it.
func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
if st.j.Manifests == nil || st.c.Try != nil {
return nil
}
req, touched := judgeRequest(st.j.Manifests.ManifestDigests(), st.c.Pulls)
if len(touched) == 0 {
return nil
}
commit, at := st.judgeHead, "the group's mesh-delivery head "+st.judgeHead
if commit == "" {
if st.j.Trunks != nil {
commit = st.j.Trunks.Heads()[DeliveryRepository]
}
at = "mesh-delivery's trunk " + commit
}
if commit == "" {
return fail(goModule, "the change's manifests cannot be judged: the trunk head of "+DeliveryRepository+" is not known")
}
// The list of predefined keys the manifests are judged against (ADR 0578 §2): the catalogue member's predefined.json
// when the change alters it, else the one the indexer keeps.
if list, from := st.predefinedList(ctx); list != nil {
req.Predefined = list
st.c.note("the manifests are judged against the predefined keys of " + from)
}
raw, err := json.Marshal(req)
if err != nil {
return fail(goModule, err.Error())
}
job := delivery.ComposeJob{Fetch: map[string]any{"repository": DeliveryRepository, "commit": commit}, Dir: JudgeDir,
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Compose: &job}
st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "")
st.c.Asked(a.ID)
if err := st.j.Asks.Put(ctx, a); err != nil {
return fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error())
}
step, err := st.j.Asks.Wait(ctx, a)
if err != nil {
return fail(goModule, "the judge of the change's manifests: "+err.Error())
}
cv, trunkCase, why := judgeAnswer(step)
if why != "" {
return fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why))
}
// A new or changed manifest is judged in full; of one the change leaves as it is, a problem the trunk already had is a
// warning (it is not refused for a rule it already broke), and a problem the change causes in it, which the trunk case
// lacks, fails the check as the change's (ADR 0579 §2).
var refused, warned []string
for _, module := range slices.Sorted(maps.Keys(cv.Manifests)) {
r := cv.Manifests[module]
if r.Accepted || r.Refused == "" {
continue
}
had := map[string]bool{}
for _, p := range strings.Split(trunkCase.Manifests[module].Refused, "\n") {
had[p] = true
}
var caused, old []string
for _, p := range strings.Split(r.Refused, "\n") {
if slices.Contains(touched, module) || !had[p] {
caused = append(caused, p)
} else {
old = append(old, p)
}
}
if len(caused) > 0 {
what := "the manifest of " + module
if module == judgeBetween {
what = "the catalogue"
}
refused = append(refused, fmt.Sprintf("the indexer's judge at %s refuses %s: %s", at, what, strings.Join(caused, "; ")))
}
if len(old) > 0 {
warned = append(warned, module+": "+strings.Join(old, "; "))
}
}
if len(warned) > 0 {
st.c.note("warnings of the indexer's judge on manifests the change leaves as they are (ADR 0579 §2): " + strings.Join(warned, " | "))
}
if len(refused) > 0 {
return fail(goModule, strings.Join(refused, "\n"))
}
st.c.note(fmt.Sprintf("the manifests of %s are accepted by the indexer's judge at %s", strings.Join(touched, ", "), at))
return nil
}
// judgeAnswer reads the judge's change and trunk cases from its step, or why it cannot.
func judgeAnswer(step delivery.Step) (change, trunk composer.CaseVerdict, why string) {
if step.State != delivery.StepPassed {
return change, trunk, fmt.Sprintf("its step %s: %s", step.State, step.Reason)
}
var a delivery.ComposeAnswer
if err := json.Unmarshal(step.Result, &a); err != nil {
return change, trunk, "its step's answer cannot be read: " + err.Error()
}
var v composer.Verdict
if err := json.Unmarshal(a.Verdict, &v); err != nil || v.V != composer.Version {
return change, trunk, fmt.Sprintf("its verdict cannot be read (version %d): %v", v.V, err)
}
found := 0
for _, c := range v.Cases {
switch c.Name {
case JudgeCase:
change, found = c, found+1
case JudgeTrunk:
trunk, found = c, found+1
}
}
if found != 2 {
return change, trunk, "its verdict does not hold both the cases " + JudgeCase + " and " + JudgeTrunk
}
return change, trunk, ""
}
// predefinedList is the list of predefined keys a case's manifests are judged against, and where it comes from: the
// catalogue member's predefined.json at its head when its merge changes it, else the one the indexer keeps; nil when
// neither can be read, said, and the judge then takes each fallback as its value declares it.
func (st *state) predefinedList(ctx context.Context) (catalogue.PredefinedList, string) {
read := func(raw []byte, from string) (catalogue.PredefinedList, string) {
list, err := catalogue.ParsePredefined(raw)
if err != nil {
st.c.note("the predefined keys of " + from + " cannot be read, so fallbacks are judged as their values declare them: " + err.Error())
return nil, ""
}
return list, from
}
if st.predefinedHead != "" && st.j.Forge != nil {
text, err := st.j.Forge.File(ctx, catalogueRepository, catalogue.PredefinedFile, st.predefinedHead)
if err == nil {
return read([]byte(text), "the catalogue member's head "+st.predefinedHead)
}
st.c.note("the catalogue member's predefined.json cannot be read: " + err.Error())
}
if st.j.Predefined == nil {
return nil, ""
}
raw, err := st.j.Predefined.PredefinedList(ctx)
if err != nil || raw == nil {
if err != nil {
st.c.note("the indexer's predefined keys cannot be read: " + err.Error())
}
return nil, ""
}
return read(raw, "the indexer's kept list")
}
// judgedByMember notes the group's mesh-delivery member head as the judge's when its merge changes the judge, and the
// catalogue member's head as where the predefined keys are read when its merge changes predefined.json.
func (st *state) judgedByMember(p Pull, merged []Fetched) {
if changesJudge(p.Repository, merged) {
st.judgeHead = p.Head
}
if p.Repository == catalogueRepository {
for _, f := range merged {
if slices.Contains(f.Changed, catalogue.PredefinedFile) {
st.predefinedHead = p.Head
}
}
}
}
@@ -38,11 +38,11 @@ func TestAPackageFailureIsNeverWordedPass(t *testing.T) { // 1901/AC5
t.Errorf("1901/AC5: the words keep %q: %q", l, v.Failed.Words)
}
}
if !strings.HasPrefix(afterHid(v.Failed.Words), c.want) {
t.Errorf("1901/AC5: words after the loud line start %q, want %q", firstLine(afterHid(v.Failed.Words)), c.want)
if !strings.HasPrefix(v.Failed.Words, c.want) {
t.Errorf("1901/AC5: words start %q, want %q", firstLine(v.Failed.Words), c.want)
}
if got := firstFailed(*v); !strings.Contains(got, "the build module hid the failure") || strings.HasSuffix(got, ": PASS") {
t.Errorf("1901/AC5, AC8: status = %q", got)
if got := firstFailed(*v); !strings.Contains(got, c.want) || strings.HasSuffix(got, ": PASS") {
t.Errorf("1901/AC5: status = %q", got)
}
})
}
@@ -63,7 +63,7 @@ func TestALongCauseIsCutToALinesBound(t *testing.T) { // 1901/AC5
func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901/AC5
for _, out := range []string{"FAIL\texample.com/timeout\t2.017s", "FAIL\texample.com/exiting\t0.006s", "FAIL\texample.com/killed\t0.006s"} {
v := levelFailure(&target{}, "integration", Tested{Failed: []Failure{{Package: "example.com/x", Output: out}}})
if !strings.HasPrefix(afterHid(v.Failed.Words), "the package failed with no test to name and no panic, signal or build error in its output") {
if !strings.HasPrefix(v.Failed.Words, "the package failed with no test to name and no panic, signal or build error in its output") {
t.Errorf("1901/AC5: %q is worded %q", out, v.Failed.Words)
}
}
@@ -74,7 +74,7 @@ func TestAPackageStoppedWithOnlyItsFailLineIsStillWorded(t *testing.T) { // 1901
func TestOnlyGosExactFormsAreSummaryOrCause(t *testing.T) { // 1901/AC5
out := " x_test.go:3: ok so far\nok then: the seat is free\n panic: child retried and recovered\npanic: the real one\nFAIL\tp\t1.0s"
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{{Package: "p", Output: out}}})
if !strings.HasPrefix(afterHid(v.Failed.Words), "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
if !strings.HasPrefix(v.Failed.Words, "panic: the real one") || !strings.Contains(v.Failed.Words, "ok then: the seat is free") {
t.Errorf("1901/AC5: words = %q", v.Failed.Words)
}
}
@@ -90,13 +90,3 @@ func TestTheWordsKeepTheirBoundExactly(t *testing.T) { // 1901/AC5
}
}
}
// afterHid is a verdict's words after the checker's loud line, which a failure with no cause from an older build module
// now starts with (1901/AC8).
func afterHid(w string) string {
if strings.HasPrefix(w, "the build module hid the failure") {
_, rest, _ := strings.Cut(w, "\n")
return rest
}
return w
}
@@ -3,17 +3,17 @@ package check
import (
"context"
"errors"
"maps"
"git.novox.be/novox/mesh-sdk/go/delivery"
"fmt"
"path"
"regexp"
"sort"
"strings"
)
// **A check builds against the group's own package heads, not against what a member names** (novox/hq ADR 0573 §2, §4;
// issue 1585). Every Go check ask carries the version of each mesh package to build against (delivery.StepAsk.Packages):
// the head of the group's member that publishes it, else its latest published version as the indexer holds it. The build
// module writes them into the workspace, whatever version or commit a member's go.mod names, so no member is refused
// for naming no version of the group's package, an older head of it, a merge commit or a commit on no pull request: that
// refusal (issue 1624's pin check) was a fault of the check (§4), and is gone.
// The pin of mesh-sdk in a Go member's go.mod (novox/hq ADR 0484 clause 9, ADR 0482 clause 2, issue 1624): a delivery group
// has one verdict, and the mesh-sdk its Go members are built and tested with is the group's: a pin to the head of the group's
// mesh-sdk pull request, or to a commit already on mesh-sdk's trunk. Any other commit is a stale or a foreign pin, and the
// group fails naming the member and the commit.
// SDKRepository is the repository of mesh-sdk, and SDKModulePath its Go module.
const (
@@ -21,47 +21,142 @@ const (
SDKModulePath = "git.novox.be/novox/mesh-sdk/go"
)
// Issue1148 is why a pin that only a head carries fails the check.
const Issue1148 = "issue 1148"
// ErrNotFound is what Forge.File wraps when the ref holds no such file (the forge answered 404): a file the ref does not hold,
// told from a forge that could not be read (issue 1845).
var ErrNotFound = errors.New("the ref holds no such file")
// Forge reads a file of a repository at a ref.
// Forge reads a file of a repository at a ref, and says whether a commit is on a repository's trunk.
type Forge interface {
// File is the text of a file at a ref (a branch, a tag or a commit).
File(ctx context.Context, repository, path, ref string) (string, error)
// OnTrunk says whether commit (a full or a 12-digit sha) is already merged into the trunk of the repository: it holds no
// commit the trunk does not.
OnTrunk(ctx context.Context, repository, commit string) (bool, error)
// FullSHA is the 40-digit sha of a commit named by its first 12 digits.
FullSHA(ctx context.Context, repository, commit string) (string, error)
// BranchHeads is the commit every branch of a repository points at, by branch name (issue 1921).
BranchHeads(ctx context.Context, repository string) (map[string]string, error)
}
// packagesOf is the version of each mesh package a Go check of repository builds against (ADR 0573 §2, §4): the latest
// published version of each package the mesh's modules use (Published), and for a package one of the case's pull
// requests publishes, that pull request's head (mesh-sdk's Go module whenever mesh-sdk is a member). A package of
// repository itself comes with the tree the check fetched, and is left out. Nil for none.
func (st *state) packagesOf(repository string) delivery.PackageVersions {
versions := delivery.PackageVersions{}
if st.j.Published != nil {
maps.Copy(versions, st.j.Published.LatestPackages())
}
if st.c.member(SDKRepository) != nil {
if _, named := versions[SDKModulePath]; !named {
versions[SDKModulePath] = ""
var pseudoVersion = regexp.MustCompile(`-([0-9a-f]{12})$`)
// SDKPin is the commit (12 hex digits) a go.mod names for mesh-sdk by a pseudo-version, or none for a go.mod that does not
// require mesh-sdk or requires a version that is no pseudo-version (a tag is a release the SDK itself names).
func SDKPin(goMod string) (commit string, ok bool) {
for _, line := range strings.Split(goMod, "\n") {
f := strings.Fields(strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "require")))
if len(f) >= 2 && f[0] == SDKModulePath {
if m := pseudoVersion.FindStringSubmatch(f[1]); m != nil {
return m[1], true
}
return "", false
}
}
for pkg := range versions {
repo, _, ok := delivery.PackageRepository(pkg)
if !ok || delivery.SameRepository(repo, repository) {
delete(versions, pkg)
continue
}
if m := st.c.member(repo); m != nil && len(m.Head) == 40 {
versions[pkg] = m.Head
}
if versions[pkg] == "" {
delete(versions, pkg)
}
}
if len(versions) == 0 {
return "", false
}
// checkPinsOf refuses a Go member whose go.mod names a mesh-sdk commit that is neither on mesh-sdk's trunk nor the head of the
// group's mesh-sdk pull request. It reads each go.mod the fetch named from the forge: the one the check builds, which is the
// head's when the head holds the file and the trunk's when it does not, since a module added on the trunk after the branch was
// cut is in the merge and not at the head (issue 1845). It answers nil when every pin is the group's.
func (st *state) checkPinsOf(ctx context.Context, t *target) verdictPtr {
if st.j.Forge == nil {
return nil
}
return versions
sdk := st.c.member(SDKRepository)
if t.module != goModule || t.pull.Repository == SDKRepository {
return nil
}
{
for _, dir := range t.fetched.Modules {
file := path.Join(dir, "go.mod")
st.c.Progress.Set(fmt.Sprintf("read %s of %s#%d", file, t.pull.Repository, t.pull.Number), t.node)
text, err := st.j.Forge.File(ctx, t.pull.Repository, file, t.pull.Head)
if errors.Is(err, ErrNotFound) && t.pull.Trunk != "" {
// Only a file the head does not hold is read at the trunk; any other failure fails the check as it did.
text, err = st.j.Forge.File(ctx, t.pull.Repository, file, t.pull.Trunk)
}
if err != nil {
return failed(forgeMachine(err, t.node), fmt.Sprintf("%s#%d: %s cannot be read to check its mesh-sdk pin: %v%s", t.pull.Repository, t.pull.Number, file, err, unheardWords(err)))
}
pin, ok := SDKPin(text)
if !ok {
continue
}
if sdk != nil && strings.HasPrefix(sdk.Head, pin) {
continue // the group's own head
}
onTrunk, err := st.j.Forge.OnTrunk(ctx, SDKRepository, pin)
if err != nil {
return failed(forgeMachine(err, t.node), fmt.Sprintf("%s#%d: whether mesh-sdk %s is on its trunk cannot be told: %v%s", t.pull.Repository, t.pull.Number, pin, err, unheardWords(err)))
}
if onTrunk {
continue
}
if st.c.Try != nil {
// A try only (issue 1921): a repository tried on a branch that mesh-sdk shares by name (a milestone's branch across
// repositories) is built with that branch's mesh-sdk head, which no group pull request holds before the pull requests exist.
branch, err := st.sameBranchPin(ctx, t, pin)
if err != nil {
return failed(forgeMachine(err, t.node), fmt.Sprintf("%s#%d: whether mesh-sdk %s is the head of a branch cannot be told: %v%s", t.pull.Repository, t.pull.Number, pin, err, unheardWords(err)))
}
if branch != "" {
st.c.note(fmt.Sprintf("pin accepted: head of mesh-sdk %s", branch))
continue
}
}
nor := "the group has no mesh-sdk pull request"
if sdk != nil {
nor = "the head of the group's mesh-sdk pull request is " + short12(sdk.Head)
}
return failed(t.node, fmt.Sprintf("%s#%d: %s names mesh-sdk %s, which is neither on mesh-sdk's trunk nor the group's head (%s)", t.pull.Repository, t.pull.Number, file, pin, nor))
}
}
return nil
}
// sameBranchPin names the branch of the tried repository that points at the tried head and that mesh-sdk also has, at the
// commit the pin names; empty when there is none.
func (st *state) sameBranchPin(ctx context.Context, t *target, pin string) (string, error) {
mine, err := st.j.Forge.BranchHeads(ctx, t.pull.Repository)
if err != nil {
return "", err
}
var same []string
for name, head := range mine {
if head == t.pull.Head && name != "main" {
same = append(same, name)
}
}
if len(same) == 0 {
return "", nil
}
sdk, err := st.j.Forge.BranchHeads(ctx, SDKRepository)
if err != nil {
return "", err
}
sort.Strings(same)
for _, name := range same {
if h, ok := sdk[name]; ok && strings.HasPrefix(h, pin) {
return name, nil
}
}
return "", nil
}
// failureVerdict is a verdict of failure the check ends with.
// pinHint is added to the failure of a Go member's fetch that names mesh-sdk while the group holds a mesh-sdk pull request:
// a pin to that pull request's head names a commit only the head carries, which go mod download cannot resolve before the
// pull request is merged (issue 1148), so such a check fails and never passes.
func pinHint(c *Case, fetchError string) string {
if c.member(SDKRepository) == nil || !strings.Contains(fetchError, "mesh-sdk") {
return ""
}
return " (a pin to the head of the group's mesh-sdk pull request names a commit only that head carries, which go mod download cannot resolve; " + Issue1148 + ")"
}
// unheardWords ends the words of a failed read of the forge that reached nobody: the forge was not heard (UnheardWords).
@@ -1,63 +1,214 @@
package check
import (
"maps"
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// published is the latest published version of each mesh package, as the indexer's modules say (Examiner.Published).
type published delivery.PackageVersions
func (p published) LatestPackages() delivery.PackageVersions {
return maps.Clone(delivery.PackageVersions(p))
// fakeForge answers the forge's two reads from files and commits it is given.
type fakeForge struct {
goMods map[string]string // `<repository> <path> <ref>` -> text
onTrunk map[string]bool // by commit prefix
err error
errAt map[string]error // by ref: a read at that ref fails with it
// branches are the branches of a repository with the commit each points at.
branches map[string]map[string]string
}
// packagesOfFetch is the package versions the fetch of repository was handed (delivery.StepAsk.Packages, issue 1585).
func packagesOfFetch(t *testing.T, b *bench, repository string) delivery.PackageVersions {
t.Helper()
for _, f := range b.steps.tool("fetch") {
if f.args["repository"] == repository {
p, err := buildstep.PackagesArg(f.args["packages"])
if err != nil {
t.Fatal(err)
}
return p
}
func (f *fakeForge) BranchHeads(_ context.Context, repository string) (map[string]string, error) {
if f.err != nil {
return nil, f.err
}
t.Fatalf("%s was not fetched", repository)
return nil
return f.branches[repository], nil
}
// ADR 0573 §4 (issue 1585 S14): a group's Go member is built against the head of the group's mesh-sdk member, whatever its
// go.mod names (an older head of it included, which issue 1624's pin check refused), and never refused for its pin; a
// pull request alone is built against the latest published version of each package, and its own repository's packages
// come with its tree.
func TestAGroupMemberIsBuiltAgainstTheSDKMembersHeadWhateverItsGoModNames(t *testing.T) { // 1624/AC2, AC4 as ADR 0573 §4 changes them
const contract = "git.novox.be/novox/mesh-delivery/contract"
func (f *fakeForge) File(_ context.Context, repository, path, ref string) (string, error) {
if f.err != nil {
return "", f.err
}
if e := f.errAt[ref]; e != nil {
return "", e
}
if s, ok := f.goMods[repository+" "+path+" "+ref]; ok {
return s, nil
}
return "", fmt.Errorf("no file %s of %s at %s: %w", path, repository, ref, ErrNotFound)
}
func (f *fakeForge) FullSHA(_ context.Context, _ string, commit string) (string, error) {
if f.err != nil {
return "", f.err
}
return commit + strings.Repeat("0", 40-len(commit)), nil
}
func (f *fakeForge) OnTrunk(_ context.Context, _ string, commit string) (bool, error) {
if f.err != nil {
return false, f.err
}
return f.onTrunk[commit[:12]], nil
}
// capturedGoMod is the go.mod the forge answered (testdata/captured/forge-get-file-go-mod.json), pinned to the commit given.
func capturedGoMod(t *testing.T, pin string) string {
t.Helper()
raw, err := os.ReadFile(filepath.Join("testdata", "captured", "forge-get-file-go-mod.json"))
if err != nil {
t.Fatal(err)
}
var a struct{ File struct{ Content string } }
if err := json.Unmarshal(raw, &a); err != nil {
t.Fatal(err)
}
return strings.Replace(a.File.Content, "cb54756c3865", pin, 1)
}
func TestTheSDKPinIsTheCommitOfAPseudoVersion(t *testing.T) { // 1624
if c, ok := SDKPin(capturedGoMod(t, "cb54756c3865")); !ok || c != "cb54756c3865" {
t.Errorf("pin = %q, %v", c, ok)
}
if _, ok := SDKPin("module x\n\nrequire git.novox.be/novox/mesh-sdk/go v0.2.0\n"); ok {
t.Error("a tagged version is no commit")
}
if c, ok := SDKPin("module x\n\nrequire git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865\n"); !ok || c != "cb54756c3865" {
t.Errorf("a one-line require: %q, %v", c, ok)
}
if _, ok := SDKPin("module x\n\ngo 1.26\n"); ok {
t.Error("a go.mod that does not require mesh-sdk has a pin")
}
}
// groupBench is a group of mesh-delivery#7 and the SDK's pull request 12, mesh-delivery's go.mod at its head naming pin.
func groupBench(t *testing.T, pin string, trunk ...string) *bench {
b := newBench(t)
b.fetch.Modules = []string{"modules/m"}
b.ex.Published = published{SDKModulePath: trunkS, contract: trunkA}
forge := &fakeForge{goMods: map[string]string{repoA + " modules/m/go.mod " + head1: capturedGoMod(t, pin)}, onTrunk: map[string]bool{}}
for _, c := range trunk {
forge.onTrunk[c] = true
}
b.ex.Forge = forge
return b
}
func TestAMemberPinnedToTheSDKMembersHeadPassesAndIsBuiltAgainstIt(t *testing.T) { // 1624/AC2, AC4
b := groupBench(t, head2[:12])
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
if v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
if got := packagesOfFetch(t, b, repoA); len(got) != 1 || got[SDKModulePath] != head2 {
t.Errorf("the member's packages = %v, want mesh-sdk at the group's head %s alone (its own contract comes with its tree)", got, head2)
var besideSDK bool
for _, f := range b.steps.tool("fetch") {
if f.args["repository"] == repoA {
refs, _ := refsOf(f.args["beside"]), true
for _, r := range refs {
besideSDK = besideSDK || (r.Repository == repoSDK && r.Commit == head2)
}
}
}
if s := besideOf(t, b, repoA)[repoSDK]; s.Commit != head2 {
if !besideSDK {
t.Error("mesh-delivery's tests were not given the SDK member at its head")
}
b = newBench(t)
b.ex.Published = published{SDKModulePath: trunkS, contract: trunkA}
if v := b.judge(t, caseOf(t, pullOf("novox/mesh-catalog", 7, head1))).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("a pull request alone: verdict = %+v", v)
}
func TestAMemberPinnedToAnOlderHeadOfTheSDKMemberFailsNamingTheMemberAndTheCommit(t *testing.T) { // 1624/AC2, AC4
b := groupBench(t, "9aaa5c80ac5c") // a commit of the SDK branch, not its head, not on the trunk
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
f := v.Failed
if v.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, repoA+"#7") || !strings.Contains(f.Words, "9aaa5c80ac5c") ||
!strings.Contains(f.Words, "neither on mesh-sdk's trunk nor the group's head") || !strings.Contains(f.Words, short12(head2)) {
t.Fatalf("verdict = %+v", v)
}
if got := packagesOfFetch(t, b, "novox/mesh-catalog"); len(got) != 2 || got[SDKModulePath] != trunkS || got[contract] != trunkA {
t.Errorf("a pull request alone is built against %v, want every package at its latest published version", got)
// Since R4g the levels run in the member's one check ask before its go.mod pins are judged (ADR 0525 §3); the verdict is
// still the pin's, whatever the levels said.
}
func TestAMemberPinnedToACommitOnTheTrunkPasses(t *testing.T) { // 1624/AC2, AC4
b := groupBench(t, "cb54756c3865", "cb54756c3865")
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
// A lone pull request of a Go member is held to the trunk alone.
b = groupBench(t, "9aaa5c80ac5c")
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "the group has no mesh-sdk pull request") {
t.Errorf("a foreign pin with no SDK member: %+v", v)
}
}
func TestAPinOnlyTheSDKHeadCarriesThatDownloadCannotResolveFailsNamingIssue1148(t *testing.T) { // 1624/AC5, ADR 0453 clause 12
b := groupBench(t, head2[:12])
b.steps.handlers["build_go_fetch"] = func(c stepCall) (any, error) {
if c.args["repository"] == repoA {
return nil, fmt.Errorf("go mod download: git.novox.be/novox/mesh-sdk/go@v0.1.14-0.20261021100209-%s: invalid version: unknown revision", head2[:12])
}
return b.fetch, nil
}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "issue 1148") || !strings.Contains(v.Failed.Words, head2[:12]) {
t.Errorf("verdict = %+v, want a failure naming issue 1148 and the commit", v)
}
}
func TestAPinThatCannotBeCheckedFailsAndNeverPasses(t *testing.T) { // ADR 0453 clause 12
b := groupBench(t, "cb54756c3865")
b.ex.Forge.(*fakeForge).err = fmt.Errorf("the forge is down")
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "the forge is down") {
t.Errorf("verdict = %+v", v)
}
}
// trunkOnlyBench is a group of mesh-delivery#7 whose fetch holds two Go modules: modules/m, which the head holds, and
// modules/new, which only the trunk holds (added on the trunk after the branch was cut). The SDK pin of each go.mod is given.
func trunkOnlyBench(t *testing.T, headPin, trunkPin string) *bench {
b := newBench(t)
b.fetch.Modules = []string{"modules/m", "modules/new"}
forge := &fakeForge{goMods: map[string]string{
repoA + " modules/m/go.mod " + head1: capturedGoMod(t, headPin),
repoA + " modules/m/go.mod " + trunkA: capturedGoMod(t, headPin),
repoA + " modules/new/go.mod " + trunkA: capturedGoMod(t, trunkPin),
}, onTrunk: map[string]bool{"cb54756c3865": true}}
b.ex.Forge = forge
return b
}
func TestAModuleOnlyTheTrunkHoldsIsReadAtTheTrunkAndPasses(t *testing.T) { // 1845/AC1
b := trunkOnlyBench(t, "cb54756c3865", "cb54756c3865")
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
}
func TestAStalePinOfAModuleOnlyTheTrunkHoldsStillFailsNamingTheFileAndTheCommit(t *testing.T) { // 1845/AC2
b := trunkOnlyBench(t, "cb54756c3865", "9aaa5c80ac5c")
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, repoA+"#7") || !strings.Contains(v.Failed.Words, "modules/new/go.mod") || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") {
t.Fatalf("verdict = %+v", v)
}
}
func TestAModuleTheHeadHoldsIsJudgedByTheHeadsPin(t *testing.T) { // 1845/AC3
b := trunkOnlyBench(t, "9aaa5c80ac5c", "cb54756c3865") // the head's pin is stale, the trunk's go.mod of the same module is fine
b.ex.Forge.(*fakeForge).goMods[repoA+" modules/m/go.mod "+trunkA] = capturedGoMod(t, "cb54756c3865")
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "modules/m/go.mod") || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") {
t.Fatalf("verdict = %+v", v)
}
}
func TestAReadThatFailsForAnotherReasonStillFailsAndIsNotTriedAtTheTrunk(t *testing.T) { // 1845/AC4
b := trunkOnlyBench(t, "cb54756c3865", "cb54756c3865")
b.ex.Forge.(*fakeForge).errAt = map[string]error{head1: fmt.Errorf("the forge answered 500")}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "modules/m/go.mod") || !strings.Contains(v.Failed.Words, "the forge answered 500") {
t.Fatalf("verdict = %+v", v)
}
}
@@ -85,13 +236,11 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: "9999999999999999999999999999999999999999"})
b.ex.Runs = runsAt{"mesh-host": hostRuns}
forge := &fakeForge{goMods: map[string]string{controller + " go.mod " + head1: capturedGoMod(t, "cb54756c3865"),
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
b.ex.Forge = forge
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
out := b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
// ADR 0548 §6, issue 568's own example: the verdict names the node-engine at the commit the mesh runs, and mesh-sdk at the
// trunk the catalogue's tests read it at; the catalogue, a member here, is not named.
if out.Beside[host] != hostRuns || out.Beside[catalogue] != "" || out.Beside[SDKRepository] != "9999999999999999999999999999999999999999" {
t.Errorf("the siblings named for the verdict = %v, want the node-engine at %s and mesh-sdk at its trunk", out.Beside, hostRuns[:12])
}
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
got := besideOf(t, b, controller)
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
@@ -99,10 +248,8 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
if h := got[host]; h.Commit != hostRuns {
t.Errorf("the node-engine beside the controller = %+v, want the commit the mesh runs %s", h, hostRuns)
}
// ADR 0573 §2: the controller is built against mesh-sdk's latest published version, so its tests read that one too,
// not the commit its go.mod names.
if s := got[SDKRepository]; s.Commit != "9999999999999999999999999999999999999999" {
t.Errorf("mesh-sdk beside the controller = %+v, want its trunk", s)
if s := got[SDKRepository]; s.Commit != "cb54756c3865"+strings.Repeat("0", 28) {
t.Errorf("mesh-sdk beside the controller = %+v, want the commit its go.mod pins, whole", s)
}
}
@@ -131,12 +278,36 @@ func TestTheNodeEnginesTestsHaveTheControllerBesideThem(t *testing.T) {
}
b = newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
b.ex.Forge = &fakeForge{goMods: map[string]string{controllerRepo + " go.mod " + head2: capturedGoMod(t, "cb54756c3865")},
onTrunk: map[string]bool{"cb54756c3865": true}}
b.judge(t, hostCase(t, pullOf(host, 5, head1), pullOf(controllerRepo, 9, head2)))
if c := besideOf(t, b, host)[controllerRepo]; c.Commit != head2 || c.Ref != "refs/pull/9/head" {
t.Errorf("the controller beside the node-engine in a group = %+v, want the member's head %s", c, head2)
}
}
func TestThePinOfTheControllerThatCannotBeReadFailsTheCheck(t *testing.T) { // review D2
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{"novox/mesh-controller": trunkA})
b.ex.Forge = &fakeForge{goMods: map[string]string{}, onTrunk: map[string]bool{}}
v := b.judge(t, caseOf(t, pullOf("novox/mesh-controller", 7, head1))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "go.mod") {
t.Errorf("verdict = %+v", v)
}
}
func TestAForgeThatIsNotHeardFailsAtTheForgeAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // 1624 C7
b := newBench(t)
b.ex.Forge = &fakeForge{err: &Unheard{Err: fmt.Errorf("nats: no responders available for request")}}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictFail || v.Failed.Machine != "forge" {
t.Fatalf("verdict = %+v, want failed at the forge", v)
}
if !IsNodesFailure(v) {
t.Errorf("a forge that was not heard is not the change's failure: %+v is not judged again when the nodes change", v.Failed)
}
}
// 1887/A12: the change's own repository is placed beside it at the trunk it is merged onto, for every repository, so a
// shrink-only list test reads the trunk's copy at $MESH_CHECK_BESIDE/<name>.
func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
@@ -153,11 +324,7 @@ func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
// ADR 0548 §6: the outcome names the siblings read at their trunk's head, never the repository it merges.
if out.Beside[SDKRepository] != trunkS || out.Beside[controllerRepo] != trunkA2 || out.Beside[catalogueRepository] != "" {
t.Errorf("the siblings named for the verdict = %v, want mesh-sdk and mesh-controller at their trunks and not the catalogue", out.Beside)
}
b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
got := besideOf(t, b, catalogueRepository)
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
@@ -200,3 +367,44 @@ func TestTheNodeEnginesCheckFailsWhereTheControllersTrunkIsNotKnown(t *testing.T
t.Fatalf("verdict = %+v", v)
}
}
// sameBranchBench is a try of mesh-delivery's feat/r6 at head1 whose go.mod pins the SDK at pin, with the SDK's feat/r6 at head2.
func sameBranchBench(t *testing.T, pin string) (*bench, *Case) {
b := groupBench(t, pin)
b.ex.Forge.(*fakeForge).branches = map[string]map[string]string{
repoA: {"main": trunkA, "feat/r6": head1},
repoSDK: {"main": "cb54756c3865" + strings.Repeat("0", 28), "feat/r6": head2},
}
c := caseOf(t, pullOf(repoA, 7, head1))
c.Try = &TryOptions{}
return b, c
}
func TestATryPinnedToTheHeadOfTheSDKBranchOfTheTriedBranchPasses(t *testing.T) { // 1921/AC1
b, c := sameBranchBench(t, head2[:12])
if v := b.judge(t, c).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
if n := c.Notes(); len(n) != 1 || n[0] != "pin accepted: head of mesh-sdk feat/r6" {
t.Errorf("notes = %q", n)
}
}
func TestATryPinnedToAnotherCommitIsStillRefused(t *testing.T) { // 1921/AC2
b, c := sameBranchBench(t, "9aaa5c80ac5c")
v := b.judge(t, c).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") || !strings.Contains(v.Failed.Words, "neither on mesh-sdk's trunk nor the group's head") {
t.Fatalf("verdict = %+v", v)
}
if len(c.Notes()) != 0 {
t.Errorf("notes = %q", c.Notes())
}
}
func TestAMergeCheckWithTheSameBranchPinIsStillRefusedWithoutAGroupPullRequest(t *testing.T) { // 1921/AC3
b, _ := sameBranchBench(t, head2[:12])
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict // no Try: a pull request's check
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "the group has no mesh-sdk pull request") {
t.Fatalf("verdict = %+v", v)
}
}
@@ -43,7 +43,7 @@ func PreviewOf(p Pull, fetched []Fetched) (delivery.Preview, bool) {
if t.Tree == "" || !digest64.MatchString(t.Manifest) {
continue
}
mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks, Paths: t.Paths, Packages: t.Packages}
mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks, Paths: t.Paths}
}
}
if merge == "" {

Some files were not shown because too many files have changed in this diff Show More