Compare commits

..
Author SHA1 Message Date
jschoubben 74ab66ca81 mesh-grants, mesh-delivery-checker: pin the group's mesh-sdk head 3f7957e (issue 774) 2026-10-27 23:02:50 +01:00
jschoubben 7f8ee816a9 Merge commit 'ac7bf01' into feat/774-vault-mints 2026-10-27 23:02:03 +01:00
jschoubben ac7bf01abf Fold the seat verbs' test into the try tools test; pin the sdk head (issue 774) 2026-10-27 23:01:18 +01:00
jschoubben fd8aba59c3 mesh-grants: grants_accept_hashes releases the clause 7 hold once, for exactly the held users of the held list, with why, said as hashes-accepted (issue 774, ADR 0561 §7, §9) 2026-10-27 22:55:14 +01:00
jschoubben cb8cab8719 Merge remote-tracking branch 'origin/main' into feat/774-vault-mints 2026-10-27 22:53:47 +01:00
jschoubben 92b6dccbec mesh-grants: pin the group's mesh-sdk head d688f10 (issue 774) 2026-10-27 22:52:30 +01:00
jschoubben 7a86c62492 mesh-delivery-checker claims its seat and serves try, try-read and try-end by the tools' own code, so the try outlives the controller (ADR 0547 §10, issue 774) 2026-10-27 22:50:32 +01:00
jschoubben afed658c69 mesh-grants: hold a list that changes several loaded users' hashes at once with no rotation by the vault naming them, and raise user-list-held naming them (issue 774, ADR 0561 §7) 2026-10-27 22:50:28 +01:00
jschoubben 3f60eb95a0 mesh-grants: grants_accounts reads the account consumers' differences in shadow, where no condition is raised (issue 774, ADR 0561 §6, §9) 2026-10-27 22:41:05 +01:00
jschoubben 23507fb3ef mesh-grants: raise a condition when an account consumer differs from its composed configuration, and bring one to it by hand with grants_consumer_match, never under the controller's lease (issue 774, ADR 0561 §6, §9) 2026-10-27 22:36:35 +01:00
jschoubben cd0ad47ad6 mesh-grants: create the durable consumers an account needs from the lease write, compared in shadow until then, so a module assigned after the controller leaves still hears what it consumes (issue 774, ADR 0561 §6) 2026-10-27 22:30:18 +01:00
jschoubben d5737f2c6d mesh-grants: compose the user list again when the vault says account-issued, so a minted account reaches the bus within one pass (issue 774, ADR 0561 §1) 2026-10-27 22:25:31 +01:00
jschoubben 04276bfda1 mesh-grants: write the bus accounts the mesh wants to a state only the vault reads, so the vault can mint them; grants_accounts reads it (issue 774, ADR 0561 §2, §9) 2026-10-27 22:24:06 +01:00
136 changed files with 3904 additions and 7294 deletions
-7
View File
@@ -1,7 +0,0 @@
# The modules of this repository that still read the controller's assignments (mesh-controller.assignments), each with
# why. mesh-grants' test (TestGrants, "assignments readers") refuses any other module.json that reads them, and an entry
# here whose module no longer does: the list only shrinks, and it is empty in the first change after the controller's
# lease is written to a holder that is no controller (novox/hq ADR 0549 §3, §6; issue 774). Each module named here
# uses the assignments only while the controller holds the lease (buscompose.LeaseHandedAway), placement alone after it.
mesh-delivery-assigner unioned with the placement until the lease write (assign.Placed with handedAway); placement alone after
mesh-grants unioned with the placement until the lease write (ReadInputs); placement alone after
@@ -18,11 +18,9 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose/judge"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provider"
@@ -72,11 +70,7 @@ func startBed(t *testing.T, leaveOut string) *bed {
if name == leaveOut {
continue
}
cfg := jetstream.KeyValueConfig{Bucket: name, History: 5}
if name == delivery.PlacementPulsesBucket {
cfg.History, cfg.LimitMarkerTTL = 1, time.Minute // a time to live per key, as the controller makes it (key-ttl)
}
kv, err := js.CreateKeyValue(ctx, cfg)
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
if err != nil {
t.Fatal(err)
}
@@ -135,7 +129,9 @@ func (b *bed) startStoppable(t *testing.T) (<-chan error, func()) {
// 1297/B2b: started whole on its credential, the assigner says on the bus every state its manifest declares, each readable,
// having opened every bucket it names. The states are read from module.json, so a state added there is expected here too.
func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) {
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
b := startBed(t, "")
done := b.start(t)
events, err := b.js.Stream(b.ctx, "EVENTS")
if err != nil {
t.Fatal(err)
@@ -177,8 +173,8 @@ func saysWhatItKeeps(t *testing.T, b *bed, done <-chan error) {
// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket.
func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
for _, missing := range readBuckets {
if missing == delivery.PlacementBucket || sourceOfGrants(missing) {
continue // it starts without it, below (the grants' source: main_test.go)
if missing == delivery.PlacementBucket {
continue // it starts without it, below
}
b := startBed(t, missing)
select {
@@ -227,8 +223,8 @@ func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
}
}
// Without the seat's placement state it starts all the same, and once the bucket is made (the controller makes
// it from the manifest) it is read with no restart: a module only the placement names is
// Review of L2 (D1): without the indexer's placement state it starts all the same, and once the bucket is made (the
// indexer and the assigner come in one delivery) it is read with no restart: a module only the placement names is
// said as placed by one source only.
pb := startBed(t, delivery.PlacementBucket)
pdone := pb.start(t)
@@ -301,6 +297,10 @@ func (b *bed) manifest(t *testing.T, module, body string) string {
// 1297/AC1: against the real bus, a fixture change touching one module on one machine is composed and answered within
// seconds under the checker's key of the check, naming the machine composed and the mesh-sdk it composed with.
func TestAChangeIsComposedOnTheBus(t *testing.T) {
// The placement answers' bound (ADR 0559 §5), shortened before the program starts and put back after it stops.
bound := assign.AnswerBound
t.Cleanup(func() { assign.AnswerBound = bound })
assign.AnswerBound = 1500 * time.Millisecond
b := startBed(t, "")
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
@@ -352,6 +352,63 @@ func TestAChangeIsComposedOnTheBus(t *testing.T) {
} else if info, err := st.Info(b.ctx); err != nil || info.State.Msgs != 0 {
t.Errorf("a group without a mesh-sdk member put %d asks on the Go build seat (%v)", info.State.Msgs, err)
}
// ADR 0559 §5 (issue 1126): a catalogue pull request that places g14 for the first time (the seed's case) is composed
// with its candidate placement, touches g14, and waits for g14's answer at its head; g14 answering the same passes it.
// A node that does not answer within the bound fails the check, named, at the bound's end with no other event; its later
// answer at the same head passes the same heads' composition.
record, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14"}})
b.putJSON(t, delivery.HaveBucket, "g14.node", record)
placed := func(group string, number int, head string) (string, uint64) {
t.Helper()
value, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: group,
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: number, Head: head},
Placement: &delivery.Placement{Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.PlacementCandidateKey(group, "g14")
rev, err := b.buckets[delivery.PlacementCandidatesBucket].Put(b.ctx, key, value)
if err != nil {
t.Fatal(err)
}
pull := []delivery.CandidatePull{{Head: delivery.Head{Repository: delivery.CatalogueRepository, Number: number, Head: head}, Modules: []string{}}}
hash, _ := delivery.HeadsHash([]delivery.Head{pull[0].Head})
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: group, HeadsHash: hash, Pulls: pull, FPSet: delivery.FingerprintSet(pull)})
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey(group)
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
key, _ = delivery.CompositionKey(group, hash)
return key, rev
}
answer := func(group, head string, rev uint64) {
t.Helper()
have, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "g14"},
Candidates: map[string]delivery.CandidateAnswer{group: {Head: head, Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"web"}}}})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, delivery.HaveBucket, "g14.node", have)
}
seedHead, lateHead := strings.Repeat("5", 40), strings.Repeat("6", 40)
seed, seedRev := placed("feat/seed", 20, seedHead)
waiting := b.verdict(t, seed, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionWaits })
if !strings.Contains(waiting.Words, "g14") {
t.Errorf("the seed's composition waits saying %q, want g14 named", waiting.Words)
}
answer("feat/seed", seedHead, seedRev)
if v := b.verdict(t, seed, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionPass }); !slices.Contains(v.Machines, "g14") {
t.Errorf("the seed's composition = %+v, want g14 composed", v)
}
late, lateRev := placed("feat/late", 21, lateHead)
failed := b.verdict(t, late, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionFail })
if failed.Machine != "g14" || !strings.Contains(failed.Words, delivery.CompositionUnanswered) {
t.Errorf("a node silent past its bound: %+v", failed)
}
answer("feat/late", lateHead, lateRev)
b.verdict(t, late, func(v delivery.CompositionVerdict) bool { return v.State == delivery.CompositionPass })
}
// get waits for a key of one of the bed's buckets and answers its value.
@@ -1005,196 +1062,3 @@ func TestAHoldWrittenAgainAfterALiftAndARestartIsAskedAgain(t *testing.T) {
}
}
}
// bedStore is the runtime's states over the bed's buckets, as the seat's verbs read and write them: the assigner's own by
// their local names, another module's `<module>.<state>`.
type bedStore struct{ b *bed }
func (s bedStore) kv(state string) jetstream.KeyValue {
if strings.Contains(state, ".") {
return s.b.buckets[strings.Replace(state, ".", "_", 1)]
}
return s.b.buckets[delivery.AssignerModule+"_"+state]
}
func (s bedStore) get(state, key string) ([]byte, uint64, bool, error) {
e, err := s.kv(state).Get(s.b.ctx, key)
if err != nil {
return nil, 0, false, nil
}
return e.Value(), e.Revision(), true, nil
}
func (s bedStore) put(state, key string, value []byte) error {
_, err := s.kv(state).Put(s.b.ctx, key, value)
return err
}
func (s bedStore) keys(state string) ([]string, error) {
keys, err := s.kv(state).Keys(s.b.ctx)
if err != nil && strings.Contains(err.Error(), "no keys") {
return nil, nil
}
return keys, err
}
func (s bedStore) del(state, key string) error { return s.kv(state).Delete(s.b.ctx, key) }
func (s bedStore) history(string, string) ([]stdio.StateRevision, error) { return nil, nil }
// 1126 (ADR 0567 §2-§6), the seat mesh-placement whole on the bus: a proposal moving web from the anchor to the laptop is
// proposed through the verb, the assigner finds both nodes must answer, they answer the same in their haves, the
// operator is asked and approves, and each key is written at a new revision while placement-changed is heard on the
// assigner's stream naming both nodes. A second proposal laid over the old revision of a key moved since is laid again,
// and declined, writes nothing and is withdrawn.
func TestAProposalIsAnsweredApprovedAndWritten(t *testing.T) {
b := startBed(t, "")
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: "MOD_" + delivery.AssignerModule + "_placement",
Subjects: []string{process.ModuleStreamSubject(delivery.AssignerModule, ">")}}); err != nil {
t.Fatal(err)
}
heard := make(chan delivery.PlacementEvent, 16)
nc, err := nats.Connect(b.srv.ClientURL())
if err != nil {
t.Fatal(err)
}
t.Cleanup(nc.Close)
if _, err := nc.Subscribe(process.ModuleStreamSubject(delivery.AssignerModule, delivery.PlacementChangedEvent), func(m *nats.Msg) {
var e delivery.PlacementEvent
if json.Unmarshal(m.Data, &e) == nil {
heard <- e
}
}); err != nil {
t.Fatal(err)
}
for _, node := range []string{"anchor", "laptop"} {
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}})
b.putJSON(t, delivery.HaveBucket, node+".node", rec)
}
b.putJSON(t, delivery.ModulesBucket, "web", []byte(`{"v":1}`))
b.putJSON(t, delivery.ModulesBucket, "shell", []byte(`{"v":1}`))
place := func(node string, modules ...string) uint64 {
p := delivery.Placement{Node: node, Modules: []delivery.PlacedModule{}}
for _, m := range modules {
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
}
raw, _ := delivery.EncodePlacement(p)
key, _ := delivery.PlacementKey(node)
rev, err := b.buckets[delivery.PlacementBucket].Put(b.ctx, key, raw)
if err != nil {
t.Fatal(err)
}
return rev
}
anchorRev, laptopRev := place("anchor", "web"), place("laptop", "shell")
bound := judge.AnswerBound
t.Cleanup(func() { judge.AnswerBound = bound })
judge.AnswerBound = 2 * time.Second
saysWhatItKeeps(t, b, b.start(t))
verbs := newSeatTools(bedStore{b}, func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"ok":true}`), nil })
propose := func() delivery.PlacementProposal {
t.Helper()
got, err := verbs.doPropose(map[string]any{"why": "move web", "changes": `[{"op":"take-off","node":"anchor","module":"web"},` +
`{"op":"place","node":"laptop","module":"web"}]`}, "operator@laptop")
if err != nil {
t.Fatal(err)
}
return got.(proposed).Proposal
}
proposalNow := func(id, what string, ok func(delivery.PlacementProposal) bool) delivery.PlacementProposal {
t.Helper()
var last delivery.PlacementProposal
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := b.buckets[delivery.PlacementProposalsBucket].Get(b.ctx, id); err == nil {
if p, err := delivery.DecodePlacementProposal(e.Value()); err == nil {
if last = p; ok(p) {
return p
}
}
}
}
t.Fatalf("the proposal %s never was %s: %+v", id, what, last)
return last
}
answerAll := func(p delivery.PlacementProposal) {
t.Helper()
for _, node := range p.Must {
key, _ := delivery.PlacementCandidateKey(p.ID, node)
e, err := b.buckets[delivery.PlacementCandidatesBucket].Get(b.ctx, key)
if err != nil {
t.Fatalf("no candidate %s: %v", key, err)
}
have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node}, Candidates: map[string]delivery.CandidateAnswer{
p.ID: {Proposal: p.ID, Revision: e.Revision(), Verdict: delivery.CandidateSame, Same: []string{"node"}}}})
b.putJSON(t, delivery.HaveBucket, node+".node", have)
}
}
p := propose()
p = proposalNow(p.ID, "told who must answer", func(p delivery.PlacementProposal) bool { return len(p.Must) > 0 })
if !slices.Equal(p.Must, []string{"anchor", "laptop"}) {
t.Fatalf("must answer = %v, want both nodes", p.Must)
}
answerAll(p)
p = proposalNow(p.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
akey, _ := delivery.PlacementAskKey(p.ID)
b.answer(t, akey, assign.OptionYes)
select {
case e := <-heard:
if e.Proposal.ID != p.ID || len(e.Nodes) != 2 || e.Nodes[0].Before != anchorRev || e.Nodes[1].Before != laptopRev ||
e.Nodes[0].After <= anchorRev || e.Warrant == nil || !slices.Equal(e.Nodes[0].Removed, []string{"web"}) {
t.Errorf("placement-changed = %+v", e)
}
case <-time.After(15 * time.Second):
t.Fatal("no placement-changed heard")
}
for node, want := range map[string][]string{"anchor": {}, "laptop": {"shell", "web"}} {
key, _ := delivery.PlacementKey(node)
e, err := b.buckets[delivery.PlacementBucket].Get(b.ctx, key)
if err != nil {
t.Fatal(err)
}
if got, err := delivery.DecodePlacement(e.Value()); err != nil || !slices.Equal(got.Assigned(), want) {
t.Errorf("%s = %v, %v, want %v", node, got.Assigned(), err, want)
}
}
if b.hasKey(delivery.PlacementProposalsBucket, p.ID) {
t.Error("an applied proposal stays in the state")
}
// A proposal whose key moves before its warrant is laid again; then declined, it writes nothing and ends.
q, err := verbs.doPropose(map[string]any{"why": "back", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
if err != nil {
t.Fatal(err)
}
moved := q.(proposed).Proposal
place("laptop", "shell", "web") // another writer moved the key
relaid := proposalNow(moved.ID, "laid again", func(p delivery.PlacementProposal) bool {
return strings.Contains(p.Words, "laid again") && len(p.Must) > 0
})
answerAll(relaid)
proposalNow(moved.ID, "asked", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalAsked })
akey, _ = delivery.PlacementAskKey(moved.ID)
b.answer(t, akey, assign.OptionNo)
for deadline := time.Now().Add(15 * time.Second); b.hasKey(delivery.PlacementProposalsBucket, moved.ID); time.Sleep(50 * time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("a declined proposal was not ended")
}
}
key, _ := delivery.PlacementKey("laptop")
if e, _ := b.buckets[delivery.PlacementBucket].Get(b.ctx, key); e == nil || !strings.Contains(string(e.Value()), "web") {
t.Error("a declined proposal wrote its key")
}
// ADR 0567 §4: a node that does not answer fails the verdict once its pulse key's time to live ends, heard on the bus
// with no timer of the assigner's, naming it.
q, err = verbs.doPropose(map[string]any{"why": "again", "changes": `[{"op":"take-off","node":"laptop","module":"web"}]`}, "op")
if err != nil {
t.Fatal(err)
}
silent := q.(proposed).Proposal
failed := proposalNow(silent.ID, "failed for a silent node", func(p delivery.PlacementProposal) bool { return p.State == delivery.ProposalFailed })
if failed.Verdict == nil || !strings.Contains(strings.Join(failed.Verdict.Fails, "; "), "laptop (offline) did not answer") {
t.Errorf("the silent node's verdict = %+v", failed.Verdict)
}
}
@@ -8,13 +8,12 @@ package main
import (
"context"
"errors"
"fmt"
"io"
"log"
"os"
"os/signal"
"slices"
"sync"
"syscall"
"time"
@@ -23,7 +22,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -51,18 +49,17 @@ func main() {
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
// holds the two lists equal), so the controller's broker grants exactly these.
var readBuckets = []string{
buscompose.ResolvedBucket,
provider.AssignmentsBucket,
delivery.BuildsBucket,
delivery.ControllerGrantsBucket,
delivery.PartGrantsBucket,
buscompose.LeaseBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ManifestsBucket,
delivery.ModulesBucket,
delivery.PlacementBucket,
delivery.PlacementCandidatesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.StepsBucket(goBuildModule),
@@ -72,27 +69,14 @@ var readBuckets = []string{
// their steps in its steps state (novox/hq ADR 0482 clause 4).
const goBuildModule = "mesh-build-go"
// ownBuckets are the assigner's own states, as its manifest declares them: the seat mesh-placement's three among them
// (ADR 0567 §1), named after the assigner as every module state is.
// ownBuckets are the assigner's own states, as its manifest declares them.
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket,
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket,
delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket}
// seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants
// them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind).
var seatBuckets = []string{delivery.PlacementBucket, delivery.PlacementProposalsBucket, delivery.PlacementCandidatesBucket,
delivery.PlacementPulsesBucket}
delivery.AssignerPutsBucket, delivery.AssignerWantsBucket}
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
// and the teller, its puts by whoever follows one (ADR 0546), and the seat mesh-placement's placement, proposals and
// candidates by the node agents, the grants part, mesh-config and the view (ADR 0567 §9), and its placement by
// the builder, the checker and the indexer, through mesh-sdk's toolchain gate (issue 774, ADR 0549 §3).
// and the teller, its puts by whoever follows one (ADR 0546).
var keptStates = []announce.State{
{Name: delivery.PlacementState, Readable: true},
{Name: delivery.PlacementProposalsState, Readable: true},
{Name: delivery.PlacementCandidatesState, Readable: true},
{Name: delivery.PlacementPulsesState, Readable: true},
{Name: "asks", Readable: true},
{Name: "compositions", Readable: true},
{Name: "differences", Readable: true},
@@ -109,47 +93,16 @@ type buckets map[string]jetstream.KeyValue
// without it and takes no put further until it starts again (ADR 0546).
func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) (buckets, error) {
out := buckets{}
// The controller's lease first (ADR 0549 §5, ADR 0561 §3): once it is handed away the seat's placement alone names
// where modules run, and the controller's assignments are no bucket the assigner needs. Not granted or not there, the
// controller holds it, as before.
away, why := false, ""
if kv, err := js.KeyValue(ctx, buscompose.LeaseBucket); err != nil {
logf("the bucket %s cannot be opened (%v): the controller is taken to hold its lease, so the assignments are read beside the placement", buscompose.LeaseBucket, err)
} else {
out[buscompose.LeaseBucket] = kv
e, gerr := kv.Get(ctx, buscompose.ControllerLeaseKey)
var raw []byte
if gerr == nil {
raw = e.Value()
}
if errors.Is(gerr, jetstream.ErrKeyNotFound) {
gerr = nil
}
away, why = buscompose.LeaseHandedAway(raw, raw != nil, gerr)
}
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
if name == buscompose.LeaseBucket {
continue
}
kv, err := js.KeyValue(ctx, name)
if name == provider.AssignmentsBucket && err != nil && away {
logf("the bucket %s cannot be opened (%v), and none is needed: %s, so the seat's placement alone is read", name, err, why)
continue
}
if name == delivery.AssignerPutsBucket && err != nil {
logf("the bucket %s cannot be opened (%v): no put is taken further until the assigner starts again", name, err)
continue
}
if slices.Contains(seatBuckets, name) && err != nil {
// The seat's states come in the same delivery as this assigner (ADR 0567 §1): without placement the controller's
// assignments alone name where modules run, as before any placement was written (ADR 0549 §5).
logf("the bucket %s cannot be opened (%v): it is bound once the bus says it was made", name, err)
continue
}
if sourceOfGrants(name) && err != nil {
// The grants part's grants and the controller's lease choose where grants are read from (issue 774): without
// them the controller's grants are read, as before the lease is handed away.
logf("the bucket %s cannot be opened (%v): the controller's grants are read until the assigner starts again", name, err)
if name == delivery.PlacementBucket && err != nil {
// The indexer's placement state comes with the second milestone (ADR 0549 §2): without it the controller's
// assignments alone name where modules run, as before any placement was read (ADR 0549 §5).
logf("the bucket %s cannot be opened (%v): the controller's assignments alone are read until the assigner starts again", name, err)
continue
}
if err != nil {
@@ -160,12 +113,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
return out, nil
}
// sourceOfGrants says the bucket is one of the two that choose the grants' source, the grants part's grants and the
// controller's lease: neither stops the assigner when it is not there (delivery.GrantSource).
func sourceOfGrants(name string) bool {
return name == delivery.PartGrantsBucket || name == buscompose.LeaseBucket
}
// run is the program: it returns when ctx ends or the bus cannot be used.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
@@ -192,14 +139,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := said.Keeps(keptStates...); err != nil {
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
}
thisNode = account.Node
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
return follow(ctx, js, conn, kvs, sayVia(ctx, js), emitting{said}, logger)
}
// thisNode is the node this process runs on, as its credential names it: said with each of the seat's events.
var thisNode string
// emitting is the process connection as the puts' caller: its calls, and its events with no options.
type emitting struct{ *process.Conn }
@@ -208,20 +151,15 @@ func (e emitting) Emit(eventType string, body any) error { return e.Conn.Emit(ev
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
var watched = []string{
buscompose.ResolvedBucket,
provider.AssignmentsBucket,
delivery.ControllerGrantsBucket,
delivery.PartGrantsBucket,
buscompose.LeaseBucket,
delivery.ControllerComposedBucket,
delivery.AnswersBucket,
delivery.BuilderBuildsBucket,
delivery.CandidatesBucket,
delivery.ModulesBucket,
delivery.PlacementBucket,
delivery.PlacementProposalsBucket,
delivery.PlacementCandidatesBucket,
delivery.PlacementPulsesBucket,
delivery.PresenceBucket,
delivery.HaveBucket,
delivery.AssignerAsksBucket,
@@ -276,13 +214,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if e.Operation() == jetstream.KeyValuePut {
value = e.Value()
}
before := w.grantsBucket()
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
if name == buscompose.LeaseBucket && w.grantsBucket() == before {
// The controller renews its lease every few seconds: only a change of where grants are read from
// makes a pass (delivery.GrantSource).
continue
}
wake()
}
}
@@ -324,7 +256,7 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
started := 0
for _, name := range watched {
if kvs[name] == nil && (slices.Contains(seatBuckets, name) || sourceOfGrants(name) || name == provider.AssignmentsBucket) {
if kvs[name] == nil && (name == delivery.PlacementBucket || name == delivery.PlacementCandidatesBucket) {
continue // not on the bus at this start (open said so)
}
started++
@@ -339,32 +271,24 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
return err
}
// The seat's states arrive with the same delivery as this assigner (ADR 0567 §1): one not on the bus at this start is
// bound once the bus says it was created, and watched from then on, with no restart (issue 1790's kind). Until then the
// controller's assignments alone name where modules run (ADR 0549 §5).
// The indexer's placement state arrives with the same delivery as this assigner (ADR 0549 §2): when it is not on the
// bus at this start it is bound once the bus says it was created, and watched from then on, with no restart (issue
// 1790's kind). Until then the controller's assignments alone name where modules run (ADR 0549 §5).
type appeared struct {
name string
kv jetstream.KeyValue
}
placementAppeared := make(chan appeared, len(seatBuckets)+2)
for _, name := range seatBuckets {
if kvs[name] == nil && js != nil {
process.AwaitBucket(ctx, js, nc, name, "the seat mesh-placement waits for it",
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
}
placementAppeared := make(chan appeared, 2)
if kvs[delivery.PlacementBucket] == nil && js != nil {
process.AwaitBucket(ctx, js, nc, delivery.PlacementBucket, "the controller's assignments alone are read",
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{delivery.PlacementBucket, kv} })
}
// And the two that choose where grants are read from, with the same delivery as the grants part that writes one.
for _, name := range []string{delivery.PartGrantsBucket, buscompose.LeaseBucket} {
if kvs[name] == nil && js != nil {
name := name
process.AwaitBucket(ctx, js, nc, name, "the controller's grants are read",
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{name, kv} })
}
// So are the placement candidates (ADR 0559 §3), with the same delivery as the indexer that writes them.
if kvs[delivery.PlacementCandidatesBucket] == nil && js != nil {
process.AwaitBucket(ctx, js, nc, delivery.PlacementCandidatesBucket, "no change's placement is composed",
logger.Printf).Ready(func(kv jetstream.KeyValue) { placementAppeared <- appeared{delivery.PlacementCandidatesBucket, kv} })
}
takePlacement := func(a appeared) error {
if kvs[a.name] != nil {
return nil // bound already, by the other of its two events
}
kvs[a.name] = a.kv
logger.Printf("the bucket %s is open and watched", a.name)
return watch(a.name)
@@ -379,16 +303,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
}
p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, said: map[string]bool{}, answeredAt: map[string]uint64{}}
p.wakeAt = wakeAtVia(ctx, wake)
p.asks = newComposeAsks(ctx, buildrun.Asker{Publisher: buildrun.JetStreamAsks{JS: js}, Waiter: buildrun.JetStreamAsks{JS: js}}, wake, logger.Printf)
if bus != nil && kvs[delivery.AssignerPutsBucket] != nil {
p.acts = newPutActs(bus, wake, logger.Printf)
}
if bus != nil {
p.seat = newSeatActs(bus, wake, logger.Printf)
}
if js != nil {
p.publish = streamPublisher(ctx, js, thisNode)
}
wake()
for {
select {
@@ -403,28 +322,36 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
case <-dirty:
reopenPuts(p)
// The grants part makes the seat's buckets in the pass that seals the list granting them, so their creation can
// be said before this credential may hear it: every pass, itself started by a watched change, binds each one the
// bus now holds, and says the seat waits while one is missing.
var unbound []string
for _, name := range seatBuckets {
if kvs[name] != nil || js == nil {
continue
}
if kv, err := js.KeyValue(ctx, name); err == nil {
if err := takePlacement(appeared{name, kv}); err != nil {
return err
}
continue
}
unbound = append(unbound, name)
}
p.sayWaits(unbound)
p.pass(ctx)
}
}
}
// wakeAtVia asks wake once at each time it is given, as a context's deadline (ADR 0437 §8: no timer), each time once.
func wakeAtVia(ctx context.Context, wake func()) func(time.Time) {
var mu sync.Mutex
asked := map[int64]bool{}
return func(at time.Time) {
mu.Lock()
defer mu.Unlock()
if asked[at.UnixNano()] {
return
}
asked[at.UnixNano()] = true
go func() {
bound, cancel := context.WithDeadline(ctx, at)
defer cancel()
<-bound.Done()
mu.Lock()
delete(asked, at.UnixNano())
mu.Unlock()
if ctx.Err() == nil {
wake()
}
}()
}
}
// PutsReopenEvery is how often, at most, a pass opens the puts' bucket again while it is not open.
var PutsReopenEvery = time.Minute
@@ -17,19 +17,13 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
func TestOpensWhatItsManifestNames(t *testing.T) {
// 1297/B2a: without a credential the program does not start, and says why.
if err := run(context.Background(), func(string) string { return "" }, io.Discard); err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
t.Errorf("run without a credential = %v", err)
}
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
if err != nil {
t.Fatal(err)
@@ -64,30 +58,20 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
// ADR 0546: whatever keeps the puts' bucket from opening (not there, not granted, no answer in time), the assigner opens
// the rest and starts, writing wants; any other bucket it cannot open stops the start, named.
// Issue 774 (ADR 0549 §5): the controller's lease not granted, the assigner starts taking it as the controller's; the
// assignments refused while the controller holds it stop the start, and once it is handed away they are not needed.
handed := []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)
for _, c := range []struct {
missing string
err error
starts bool
lease []byte
said string
}{
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true, nil, "no put is taken further"},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true, nil, "no put is taken further"},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true, nil, "no put is taken further"},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false, nil, ""},
// Issue 774: the grants part's grants and the controller's lease choose the grants' source; neither stops the start.
{delivery.PartGrantsBucket, jetstream.ErrBucketNotFound, true, nil, "the controller's grants are read"},
{buscompose.LeaseBucket, errors.New("nats: permissions violation"), true, nil, "taken to hold its lease"},
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), false, []byte(`{"instance":"novox/1"}`), ""},
{provider.AssignmentsBucket, errors.New("nats: permissions violation"), true, handed, "the seat's placement alone is read"},
{delivery.AssignerPutsBucket, jetstream.ErrBucketNotFound, true},
{delivery.AssignerPutsBucket, errors.New("nats: permissions violation for subscription to \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_puts\""), true},
{delivery.AssignerPutsBucket, context.DeadlineExceeded, true},
{delivery.AssignerWantsBucket, errors.New("nats: permissions violation"), false},
} {
var logged string
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err, lease: c.lease}, func(f string, a ...any) { logged += fmt.Sprintf(f, a...) })
kvs, err := open(context.Background(), refusingJS{missing: c.missing, err: c.err}, func(f string, a ...any) { logged = fmt.Sprintf(f, a...) })
switch {
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, c.said)):
case c.starts && (err != nil || kvs[c.missing] != nil || len(kvs) != len(readBuckets)+len(ownBuckets)-1 || !strings.Contains(logged, "no put is taken further")):
t.Errorf("%s refused with %v: the start answered %v, %d buckets, logged %q", c.missing, c.err, err, len(kvs), logged)
case !c.starts && (err == nil || !strings.Contains(err.Error(), "grant of "+c.missing)): // 1945: named with its grant
t.Errorf("%s refused with %v: the start answered %v", c.missing, c.err, err)
@@ -121,23 +105,26 @@ func TestOpensWhatItsManifestNames(t *testing.T) {
}
}
// 1297/B2a: without a credential the program does not start, and says why.
func TestNoCredential(t *testing.T) {
err := run(context.Background(), func(string) string { return "" }, io.Discard)
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
t.Errorf("run without a credential = %v", err)
}
}
// refusingJS opens every bucket but one, which it refuses with err.
type refusingJS struct {
jetstream.JetStream
missing string
err error
lease []byte // the controller's lease, when it holds one
}
func (r refusingJS) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
if name == r.missing {
return nil, r.err
}
kv := &memKV{values: map[string][]byte{}}
if name == buscompose.LeaseBucket && r.lease != nil {
kv.values[buscompose.ControllerLeaseKey] = r.lease
}
return kv, nil
return &memKV{values: map[string][]byte{}}, nil
}
// watchKV is a memKV whose watch hands its entries and then the marker that all stands was handed, or ends at once when
@@ -40,20 +40,12 @@ type passer struct {
// noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is
// said in the journal when it appears or changes, and its clearing when the node has a want again; one line per change.
noWant map[string]string
// unreached is, by node and module, the words last said why a wanted module may not be reached there (issue 1960: its
// provider is down, which holds no want): said once per words, and once more when it is reached again.
unreached map[string]string
// nodeFailed is, by node and module, the words of the condition last raised for a failure that is the node's own (ADR 0557
// §1, issue 1496): raised once per words, cleared once when it ends.
nodeFailed map[string]string
// The seat mesh-placement's holder (seat.go, ADR 0567): its calls of the controller (nil: none is made), how it says
// the seat's events (nil: none is said), the proposals it said, and what Intended computed by proposal and base.
seat *seatActs
publish publisher
saidProposed map[string]bool
// saidWaits is the last said of the seat's wait for its states (sayWaits).
saidWaits string
intended map[string]map[string]map[string]bool
// wakeAt asks a pass at a time: the end of a placement candidate's answer bound (ADR 0559 §5), as a context's deadline
// (ADR 0437 §8: no timer). Nil asks none.
wakeAt func(at time.Time)
}
func (p *passer) clock() time.Time {
@@ -64,7 +56,6 @@ func (p *passer) clock() time.Time {
}
func (p *passer) pass(ctx context.Context) {
p.placementSeat(ctx)
p.compositions(ctx)
p.wants(ctx)
p.puts(ctx)
@@ -129,7 +120,7 @@ func (p *passer) wants(ctx context.Context) {
}
}
wanted := p.w.wantsOf()
frozen := p.sayPlacement(p.withCalls(p.w.placementDiffers()))
frozen := p.sayPlacement(p.w.placementDiffers())
placed := p.w.placedNodes()
puts := p.putsByModule()
p.staleAsks(ctx, trunk)
@@ -149,8 +140,6 @@ func (p *passer) wants(ctx context.Context) {
stale := map[string]neitherOne{}
left := map[string]string{}
defer func() { p.sayNeither(neither, stale, left) }()
// Where grants are read from, once for the pass: the grants part's while the lease is handed away (issue 774).
grants := p.w.grantsBucket()
for name := range names {
fp := trunk[name].FP2
name := name
@@ -166,7 +155,7 @@ func (p *passer) wants(ctx context.Context) {
},
GrantedAt: func(fp string) bool {
key, err := delivery.ControllerGrantKey(name, fp)
return err == nil && p.w.has(grants, key)
return err == nil && p.w.has(delivery.ControllerGrantsBucket, key)
},
Restored: func(node string) bool {
key, err := delivery.RestoreAskKey(node, name, fp)
@@ -182,8 +171,8 @@ func (p *passer) wants(ctx context.Context) {
hold, _ := delivery.HoldAskKey(name, fp)
m.Released = p.answeredWith(hold, "release")
m.Kept = p.answeredWith(hold, "keep")
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(grants, grant)
if raw := p.w.value(grants, grant); raw != nil {
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held) && !m.Released, p.w.has(delivery.ControllerGrantsBucket, grant)
if raw := p.w.value(delivery.ControllerGrantsBucket, grant); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
m.GrantedOn = func(node string) bool { return assign.GrantNamesNode(g, node, name) }
}
@@ -201,7 +190,6 @@ func (p *passer) wants(ctx context.Context) {
gatherNeither(neither, name, out)
leftNeither(left, m, wanted[name], out)
p.sayNoWant(name, m.Nodes, wanted[name], out)
p.sayUnreached(name, m.Nodes, out)
p.sayNodeFailed(name, fp, m.Nodes, nodes, out, first)
if first {
startClears(stale, m, out)
@@ -280,26 +268,6 @@ func (p *passer) sayNoWant(module string, nodes []string, before map[string]deli
}
}
// sayUnreached says in the journal, once per words, that a module is wanted on a node and may not be reached there, since
// no provider of what it requires runs healthy (rule 5, which holds no want: issue 1960), and once when it is reached again.
func (p *passer) sayUnreached(module string, nodes []string, out assign.Outcome) {
if p.unreached == nil {
p.unreached = map[string]string{}
}
for _, node := range nodes {
key := node + "." + module
why, last := out.Unreached[node], p.unreached[key]
switch {
case why != "" && why != last:
p.logf("%s on %s is wanted and may not be reached: %s", module, node, why)
p.unreached[key] = why
case why == "" && last != "":
p.logf("%s on %s is reached: its providers run healthy", module, node)
delete(p.unreached, key)
}
}
}
// sayNodeFailed raises, once per node and words, the condition of a failed health check of a module's newest fingerprint fp
// that is the node's own, since fp runs healthy elsewhere and is not held (ADR 0557 §1, issue 1496), and says it in the journal;
// it clears the condition once when the node no longer fails alone, or is no longer one of the module's nodes. The first pass
@@ -514,6 +482,9 @@ func (p *passer) compositions(ctx context.Context) {
} else {
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
}
if ch.Placement != nil && v.State != delivery.CompositionWaits {
v = p.answers(*mesh, ch, v)
}
}
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
before.Took, before.V = v.Took, 0
@@ -525,6 +496,40 @@ func (p *passer) compositions(ctx context.Context) {
}
}
// answers judges every placed node's answer to the change's placement into v (ADR 0559 §5). While a node has not answered
// within the bound the verdict waits, and the pass at the bound's end is asked (wakeAt), so a silent node is failed then,
// named, and a later answer of the same head is judged on its own have's change.
func (p *passer) answers(mesh assign.Mesh, ch assign.Change, v delivery.CompositionVerdict) delivery.CompositionVerdict {
trunkPlaced := map[string]delivery.Placement{}
for key, value := range p.w.read(delivery.PlacementBucket) {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
continue
}
if pl, err := delivery.DecodePlacement(value); err == nil && pl.Node == node {
trunkPlaced[node] = pl
}
}
fps := map[string]string{}
for name, m := range p.w.trunk() {
if m.FP2 != "" {
fps[name] = m.FP2
} else if m.FP != "" {
fps[name] = m.FP
}
}
intended, notes := assign.Intended(mesh.Input, trunkPlaced, ch.Placement, fps)
a := assign.JudgeAnswers(ch.Placement, assign.PlacedNodes(trunkPlaced, ch.Placement), p.w.candidateAnswers(ch.Group), intended, p.w.online(), p.clock())
v = assign.WithAnswers(v, a)
if v.State != delivery.CompositionWaits {
v.Warnings = append(v.Warnings, notes...)
}
if a.Waits && p.wakeAt != nil {
p.wakeAt(a.Until)
}
return v
}
// put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass.
func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]byte, error)) {
value, err := encode()
@@ -13,7 +13,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
@@ -136,9 +135,6 @@ func (m *memKV) Update(_ context.Context, key string, value []byte, revision uin
type memStore map[string]*memKV
func (s memStore) get(state, key string) ([]byte, uint64, bool, error) {
if s[state] == nil {
return nil, 0, false, nil
}
v, ok := s[state].values[key]
return v, uint64(len(v)), ok, nil
}
@@ -857,7 +853,7 @@ func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
p.sayPlacement(p.w.placementDiffers())
if !slices.Equal(got["g14"], []string{"a"}) || !frozen["g14.b"] || !frozen["g14.c"] || frozen["g14.a"] || len(said) != 2 ||
said[0].State != asks.StateRaised || !strings.Contains(said[0].Explanation, "the controller's assignments names b on g14") ||
!strings.Contains(said[1].Explanation, "the seat mesh-placement's placement names c on g14") {
!strings.Contains(said[1].Explanation, "the catalogue's placement names c on g14") {
t.Fatalf("one source alone: assigned %v, frozen %v, said %+v", got, frozen, said)
}
place("g14", "a", "b", "c")
@@ -877,24 +873,6 @@ func aPairOneSourceNamesAloneGetsNoWantAndOneCondition(t *testing.T) {
if m.Input == nil || len(m.Input.Nodes) != 1 || !slices.Equal(m.Input.Nodes[0].Assigned, []string{"a", "b", "c"}) || m.Input.Nodes[0].Place.Key != "k" {
t.Errorf("the check's input from the bus: %+v (%q)", m.Input, m.InputUnread)
}
// Issue 774 (ADR 0549 §5, §6): once the controller's lease is handed away the placement alone names where modules
// run: a pair in the placement only gets its want and no condition, and one in the assignments only is no module of
// the node. A lease that goes back to a controller brings the comparison back.
place("n2", "d")
assigned("g14", "e")
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`))
got, all := p.w.assigned()
if !slices.Equal(got["n2"], []string{"d"}) || !slices.Equal(got["g14"], []string{"a", "b", "c"}) || len(all) != 0 {
t.Errorf("handed away: assigned %v, assignments %v; want the placement alone", got, all)
}
if frozen := p.sayPlacement(p.w.placementDiffers()); len(frozen) != 0 || len(said) != 4 {
t.Errorf("handed away: frozen %v, said %+v; want no condition", frozen, said[4:])
}
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(`{"instance":"novox/1","epoch":58}`))
if frozen := p.sayPlacement(p.w.placementDiffers()); !frozen["n2.d"] || !frozen["g14.e"] {
t.Errorf("a controller's lease again: frozen %v; want both pairs held", frozen)
}
}
// 1126 (ADR 0547 §2 to §4): a node the catalogue places composes its own: its wants name the build alone and its own want
@@ -213,7 +213,8 @@ func (p *passer) putWorld(node, module string, put delivery.Put, manifest catalo
// composedFor is the composition of module for node at fp and whether the grant at fp names the node's accounts: the one
// place a put reads them (ADR 0546 clause 4). Today their writer is the controller (mesh-controller_composed and
// mesh-controller_grants); the grants are the grants part's while the controller's lease is handed away (grantsBucket).
// mesh-controller_grants); when composing and grants move out of the controller into delivery's own parts, this reads them
// from their new owner and nothing else changes.
func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest, bool) {
var out *delivery.ComposedDigest
if d, ok := p.w.composed(node, module, fp); ok {
@@ -221,7 +222,7 @@ func (p *passer) composedFor(node, module, fp string) (*delivery.ComposedDigest,
}
granted := false
if gkey, err := delivery.ControllerGrantKey(module, fp); err == nil {
if raw := p.w.value(p.w.grantsBucket(), gkey); raw != nil {
if raw := p.w.value(delivery.ControllerGrantsBucket, gkey); raw != nil {
if g, err := delivery.DecodeControllerGrant(raw); err == nil {
granted = g.NamesNode(node, module)
}
@@ -1,810 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"reflect"
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/compose/judge"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-assigner/internal/assign"
)
// The seat mesh-placement's holder in the assigner's process (novox/hq ADR 0567 §2-§8): each pass follows every open
// proposal one step, from what the world holds. A proposal the verbs wrote is said (placement-proposed); the nodes it
// must hear from are found by composing every node with and without it, and those its key does not change get a reached
// candidate; their answers are judged (placement-judged) and, on a passing verdict, the operator is asked to approve it;
// on the warrant each changed key is written by compare-and-set at the revision it was laid over (placement-changed),
// and the controller's assignments are kept in step through its verbs until the lease write. A key moved meanwhile
// lays the proposal again; a decline, an expiry or a withdrawal ends it (placement-withdrawn). Every event is said on the
// assigner's own stream, so only this process says them. Nothing polls: the wake at an answer's bound is the one wake.
// publisher says one of the seat's events on the assigner's stream, under a message id the bus keeps one of.
type publisher func(event string, body any, id string) error
// streamPublisher publishes on mesh.stream.mesh-delivery-assigner.event.<event>, with the headers an emit carries.
func streamPublisher(ctx context.Context, js jetstream.JetStream, node string) publisher {
return func(event string, body any, id string) error {
data, err := json.Marshal(body)
if err != nil {
return err
}
msg := nats.NewMsg(process.ModuleStreamSubject(delivery.AssignerModule, event))
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", delivery.AssignerModule)
msg.Header.Set("x-node", node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
_, err = js.PublishMsg(ctx, msg, jetstream.WithMsgID(id))
return err
}
}
// seatActs keeps what the seat's calls of the controller came to while this process runs: the export read again at a
// seed's warrant, and each call that keeps the controller's assignments in step (§8).
type seatActs struct {
mu sync.Mutex
bus caller
wake func()
logf func(string, ...any)
export map[string]exportCheck
// calls is, by pair (node and module, or node and provision for a pin), what the last call for it came to.
calls map[string]string
}
// exportCheck is the export read again for one proposal: running, its answer, or why it was not read.
type exportCheck struct {
running bool
export map[string]delivery.Placement
err string
}
func newSeatActs(bus caller, wake func(), logf func(string, ...any)) *seatActs {
return &seatActs{bus: bus, wake: wake, logf: logf, export: map[string]exportCheck{}, calls: map[string]string{}}
}
// exportOf is the export read again for a proposal, asked once in the background.
func (a *seatActs) exportOf(id string) exportCheck {
a.mu.Lock()
defer a.mu.Unlock()
if c, ok := a.export[id]; ok {
return c
}
a.export[id] = exportCheck{running: true}
go func() {
defer a.wake()
raw, err := a.bus.Call(exportCall, map[string]any{})
c := exportCheck{}
if err != nil {
c.err = "the controller's placement-export did not answer: " + err.Error()
} else if c.export, err = assign.ExportOf(raw); err != nil {
c.err = err.Error()
}
a.mu.Lock()
a.export[id] = c
a.mu.Unlock()
}()
return exportCheck{running: true}
}
// forget drops what is kept of a proposal's export, so a later warrant reads it again.
func (a *seatActs) forget(id string) {
if a == nil {
return
}
a.mu.Lock()
delete(a.export, id)
a.mu.Unlock()
}
// record records once, in the controller's hand-act log, that placement was written on the operator's warrant (§5): the
// router's record of the teller's ask says who chose and how (ADR 0274).
func (a *seatActs) record(warrant string) {
if a == nil || warrant == "" {
return
}
ask, _, _ := strings.Cut(warrant, "@")
go func() {
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+".warranted", map[string]any{"asker": delivery.TellerModule, "ask": ask}); err != nil {
a.logf("the placement's warrant %s is not recorded in the hand-act log: %v", ask, err)
}
}()
}
// handAct records a hand-back in the controller's hand-act log (§7).
func (a *seatActs) handAct(prop delivery.PlacementProposal) {
go func() {
if _, err := a.bus.Call(handActCall, map[string]any{"what": "mesh-placement hand-back of " + strings.Join(prop.Nodes(), ", "),
"why": prop.Why, "cause": "asked by " + prop.By + ", approved by the operator"}); err != nil {
a.logf("the hand-back %s is not recorded in the hand-act log: %v", prop.ID, err)
}
}()
}
// callAll makes the controller's calls in order in the background, each said in the journal and kept by its pair.
func (a *seatActs) callAll(calls []assign.ControllerCall) {
if len(calls) == 0 {
return
}
a.mu.Lock()
for _, c := range calls {
a.calls[c.Node+"."+c.Module] = c.Key() + " is asked"
}
a.mu.Unlock()
go func() {
defer a.wake()
for _, c := range calls {
words := c.Key() + " answered"
if _, err := a.bus.Call("seat:"+catalogue.ControllerSeatName+"."+c.Verb, c.Args); err != nil {
words = c.Key() + " refused: " + err.Error()
}
a.logf("keeping the controller in step with placement: %s", words)
a.mu.Lock()
a.calls[c.Node+"."+c.Module] = words
a.mu.Unlock()
}
}()
}
// callOf is what the last call for a pair came to, "" when none was made.
func (a *seatActs) callOf(node, module string) string {
if a == nil {
return ""
}
a.mu.Lock()
defer a.mu.Unlock()
return a.calls[node+"."+module]
}
// withCalls names, on each pair the two sources do not both name, the call the assigner made for it (§8).
func (p *passer) withCalls(pairs []assign.Pair) []assign.Pair {
for i := range pairs {
pairs[i].Call = p.seat.callOf(pairs[i].Node, pairs[i].Module)
}
return pairs
}
// sayWaits raises, once, that the seat waits for the states not yet bound, and clears it once, when all are.
func (p *passer) sayWaits(unbound []string) {
c := assign.SeatWaitsCondition(unbound)
state := string(c.State) + c.Summary
if p.say == nil || p.said[c.Key] && p.saidWaits == state {
return
}
if c.State == asks.StateCleared && p.saidWaits == "" {
return // nothing was raised by this process
}
if err := p.say(c); err != nil {
p.logf("that the seat waits for its states could not be said, and is said at the next pass: %v", err)
return
}
p.said[c.Key], p.saidWaits = true, state
}
// seatPlacements is every node's placement key and its revision, as the watch holds them.
func (p *passer) seatPlacements() (map[string]delivery.Placement, map[string]uint64) {
out, revs := map[string]delivery.Placement{}, map[string]uint64{}
for key, value := range p.w.read(delivery.PlacementBucket) {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
continue
}
if pl, err := delivery.DecodePlacement(value); err == nil && pl.Node == node {
out[node], revs[node] = pl, p.w.metaOf(delivery.PlacementBucket, key).revision
}
}
return out, revs
}
// candidatesOf is a proposal's candidates by node, with their keys' revisions and the newest write.
func (p *passer) candidatesOf(id string) (map[string]delivery.PlacementCandidate, map[string]uint64, time.Time) {
out, revs := map[string]delivery.PlacementCandidate{}, map[string]uint64{}
var newest time.Time
for key, value := range p.w.read(delivery.PlacementCandidatesBucket) {
pid, node, err := delivery.ParsePlacementCandidateKey(key)
if err != nil || pid != id {
continue
}
c, err := delivery.DecodePlacementCandidate(value)
if err != nil {
continue
}
meta := p.w.metaOf(delivery.PlacementCandidatesBucket, key)
out[node], revs[node] = c, meta.revision
if meta.at.After(newest) {
newest = meta.at
}
}
return out, revs, newest
}
// answersTo is, by node, each node's answer to the proposal in its `<node>.node` have.
func (p *passer) answersTo(id string) map[string]*delivery.CandidateAnswer {
out := map[string]*delivery.CandidateAnswer{}
for key, value := range p.w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
h, err := delivery.DecodeHaveNode(value)
if err != nil {
continue
}
if a, ok := h.Candidates[id]; ok {
out[node] = &a
}
}
return out
}
// placementSeat takes every open proposal one step further.
func (p *passer) placementSeat(ctx context.Context) {
for _, b := range seatBuckets {
if p.kvs[b] == nil {
return
}
}
if p.saidProposed == nil {
p.saidProposed, p.intended = map[string]bool{}, map[string]map[string]map[string]bool{}
}
current, revs := p.seatPlacements()
ids := make([]string, 0)
raws := p.w.read(delivery.PlacementProposalsBucket)
for id := range raws {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
prop, err := delivery.DecodePlacementProposal(raws[id])
if err != nil || prop.ID != id {
p.logf("the proposal %s cannot be read, so it is left: %v", id, err)
continue
}
p.stepProposal(ctx, prop, p.w.metaOf(delivery.PlacementProposalsBucket, id).revision, current, revs)
}
}
// pulse writes a bound's key with its time to live (§4): its expiry, heard on the watch as the server's delete marker,
// is the event that ends the wait. Written again over one that stands, so its time starts anew.
func (p *passer) pulse(ctx context.Context, key string, ttl time.Duration) {
kv := p.kvs[delivery.PlacementPulsesBucket]
_ = kv.Delete(ctx, key)
rev, err := kv.Create(ctx, key, []byte(`{}`), jetstream.KeyTTL(ttl))
if err != nil {
p.logf("the bound %s is not written, so its wait ends at once: %v", key, err)
return
}
p.w.putEntry(delivery.PlacementPulsesBucket, key, []byte(`{}`), rev, p.clock())
}
// pulsing says whether a bound's key stands.
func (p *passer) pulsing(key string) bool { return p.w.has(delivery.PlacementPulsesBucket, key) }
// deletePulses deletes every bound of a proposal.
func (p *passer) deletePulses(ctx context.Context, id string) {
for key := range p.w.read(delivery.PlacementPulsesBucket) {
if strings.HasSuffix(key, "."+id) || strings.Contains(key, "."+id+".") {
if err := p.kvs[delivery.PlacementPulsesBucket].Delete(ctx, key); err == nil {
p.w.put(delivery.PlacementPulsesBucket, key, nil)
}
}
}
}
// ask raises (or raises again) a proposal's authorising ask and its bound, and answers the proposal asked.
func (p *passer) ask(ctx context.Context, prop delivery.PlacementProposal, akey, brief string) (delivery.PlacementProposal, bool) {
a, err := assign.PlacementAsk(akey, prop, brief, assign.PlacementAsking(prop, p.clock()), p.clock())
if err != nil {
p.logf("the ask about the proposal %s is not written: %v", prop.ID, err)
return prop, false
}
p.put(ctx, delivery.AssignerAsksBucket, akey, func() ([]byte, error) { return delivery.EncodeAssignerAsk(akey, a) })
if pk, err := delivery.PulseAskKey(prop.ID); err == nil {
p.pulse(ctx, pk, a.Ask.Expires.Sub(p.clock()))
}
prop.State, prop.Ask = delivery.ProposalAsked, a.Ask.ID
return prop, true
}
// stepProposal takes one proposal one step further.
func (p *passer) stepProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64) {
if !p.saidProposed[prop.ID] && prop.State == delivery.ProposalAnswering && prop.Verdict == nil {
p.sayEvent(delivery.PlacementProposedEvent, delivery.PlacementEvent{Proposal: prop}, prop.ID)
p.saidProposed[prop.ID] = true
}
if prop.Withdrawn != "" {
p.endProposal(ctx, prop, prop.Withdrawn)
return
}
akey, _ := delivery.PlacementAskKey(prop.ID)
again := !prop.Again.IsZero() && (prop.Verdict == nil || prop.Again.After(prop.Verdict.At))
if prop.State == delivery.ProposalAsked {
askPulse, _ := delivery.PulseAskKey(prop.ID)
switch {
case p.answeredWith(akey, assign.OptionYes):
if prop.HandBack {
p.handBack(ctx, prop, akey, revs)
} else {
p.warranted(ctx, prop, rev, akey, current, revs)
}
return
case p.answeredWith(akey, assign.OptionNo):
p.endProposal(ctx, prop, "declined by the operator")
return
case !p.pulsing(askPulse) && !again:
p.endProposal(ctx, prop, "its ask expired unanswered")
return
}
}
if prop.HandBack {
// The operator's alone (§7): no node answers it; its ask is raised at once, and again on again.
if prop.State != delivery.ProposalAsked || again {
brief := fmt.Sprintf("%s (asked by %s): every node's placement key is deleted, %s, and each node returns to the "+
"controller's compositions; the history is kept", prop.Why, prop.By, strings.Join(prop.Nodes(), ", "))
now := p.clock().UTC()
prop.Verdict = &delivery.PlacementVerdict{State: delivery.VerdictPass, At: now}
if prop, ok := p.ask(ctx, prop, akey, brief); ok {
p.writeProposal(ctx, prop, rev)
}
}
return
}
if moved(prop, revs) {
p.layAgain(ctx, prop, rev, current, revs, "a key it was laid over moved since")
return
}
cands, candRevs, written := p.candidatesOf(prop.ID)
if len(prop.Must) == 0 {
p.mustAnswer(ctx, prop, rev, current, cands)
return
}
for _, node := range prop.Must {
if _, ok := cands[node]; !ok {
return // a candidate just written is not heard yet: its watch wakes the next pass
}
}
if written.After(prop.Written) {
prop.Written = written
}
pulsing := map[string]bool{}
for _, node := range prop.Must {
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
pulsing[node] = p.pulsing(k)
}
}
v := judge.JudgeAnswers(prop, p.intendedOf(prop, current, cands), p.answersTo(prop.ID), candRevs, p.w.online(), pulsing, p.clock())
if prop.Verdict != nil && sameVerdict(*prop.Verdict, v) && !again {
return
}
prop.Verdict = &v
switch v.State {
case delivery.VerdictFail:
prop.State, prop.Ask = delivery.ProposalFailed, ""
case delivery.VerdictWaits:
prop.State, prop.Ask = delivery.ProposalAnswering, ""
case delivery.VerdictPass:
if prop.State != delivery.ProposalAsked || again {
var ok bool
if prop, ok = p.ask(ctx, prop, akey, judge.Brief(prop, v)); !ok {
return
}
if again {
prop.Words = "judged and asked again (again)"
}
}
}
if p.writeProposal(ctx, prop, rev) {
p.sayEvent(delivery.PlacementJudgedEvent, delivery.PlacementEvent{Proposal: prop},
fmt.Sprintf("%s.judged.%d", prop.ID, v.At.UnixNano()))
}
}
// sameVerdict says two verdicts say the same, their time aside.
func sameVerdict(a, b delivery.PlacementVerdict) bool {
a.At, b.At = time.Time{}, time.Time{}
return reflect.DeepEqual(a, b)
}
// moved says whether a key the proposal changes is no longer at the revision it was laid over.
func moved(prop delivery.PlacementProposal, revs map[string]uint64) bool {
for node, base := range prop.Base {
if revs[node] != base {
return true
}
}
return false
}
// intended is, by proposal and the revisions it was laid over, what Intended computed, kept while this process runs.
func (p *passer) intendedOf(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) map[string]map[string]bool {
key := fmt.Sprint(prop.ID, prop.Base)
if got, ok := p.intended[key]; ok {
return got
}
got, _, _ := p.compute(prop, current, cands)
p.intended[key] = got
return got
}
// compute composes every node with the current placement and with the proposal laid over it (judge.Intended).
func (p *passer) compute(prop delivery.PlacementProposal, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) (map[string]map[string]bool, []string, []string) {
laid := map[string]*delivery.Placement{}
for _, node := range prop.Nodes() {
if c, ok := cands[node]; ok && !c.Reached {
laid[node] = c.Placement
}
}
get := kvGetter(p.kvs[delivery.ManifestsBucket])
r := worldReader{w: p.w, manifests: get}
var now, with *compose.Input
if in, err := frombus.Input(context.Background(), r, frombus.Mesh{}, frombus.Scope{Every: true}); err == nil {
now = &in
}
if in, err := frombus.Input(context.Background(), r, frombus.Mesh{}, frombus.Scope{Every: true, Placements: laid}); err == nil {
with = &in
}
nodes := map[string]bool{}
for n := range current {
nodes[n] = true
}
if records, err := r.Records(context.Background()); err == nil {
for n := range records {
nodes[n] = true
}
}
names := make([]string, 0, len(nodes))
for n := range nodes {
names = append(names, n)
}
fps := map[string]string{}
for name, m := range p.w.trunk() {
if m.FP2 != "" {
fps[name] = m.FP2
}
}
return judge.Intended(now, with, current, laid, names, fps)
}
// mustAnswer finds the nodes the proposal must hear from (§3), writes a reached candidate for each whose key it does not
// change, and writes them into the proposal.
func (p *passer) mustAnswer(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, cands map[string]delivery.PlacementCandidate) {
for _, node := range prop.Nodes() {
if _, ok := cands[node]; !ok {
return // the verbs write the candidates before the proposal: wait to hear them
}
}
intended, must, notes := p.compute(prop, current, cands)
p.intended[fmt.Sprint(prop.ID, prop.Base)] = intended
for _, node := range must {
if _, ok := cands[node]; ok {
continue
}
key, err := delivery.PlacementCandidateKey(prop.ID, node)
if err != nil {
continue
}
c := delivery.PlacementCandidate{Proposal: prop.ID, Reached: true}
if pl, ok := current[node]; ok {
c.Placement = &pl
}
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
}
for _, node := range must {
if k, err := delivery.PulseAnswerKey(prop.ID, node); err == nil {
p.pulse(ctx, k, judge.AnswerBound)
}
}
prop.Must = must
if len(notes) > 0 {
prop.Words = strings.TrimPrefix(prop.Words+"; "+strings.Join(notes, "; "), "; ")
}
p.writeProposal(ctx, prop, rev)
}
// layAgain lays a proposal over the keys as they stand now (§5: a key moved): its candidates are written again, the
// nodes that must answer found again, and it is judged and asked again; a proposal that no longer lays ends.
func (p *passer) layAgain(ctx context.Context, prop delivery.PlacementProposal, rev uint64, current map[string]delivery.Placement, revs map[string]uint64, why string) {
held, recorded := p.lays(ctx)
again, laid, findings := assign.Propose(assign.ProposeIn{ID: prop.ID, By: prop.By, Why: prop.Why, At: prop.At, Changes: prop.Changes,
Current: current, Revisions: revs, Held: held, Recorded: recorded,
Seed: prop.Seed, ExportDigest: prop.ExportDigest, Leased: p.w.handedAway()})
if len(findings) > 0 {
p.endProposal(ctx, prop, why+", and laid over the keys now it is refused: "+strings.Join(findings, "; "))
return
}
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
for _, node := range again.Nodes() {
key, _ := delivery.PlacementCandidateKey(prop.ID, node)
c := delivery.PlacementCandidate{Proposal: prop.ID, Placement: laid[node]}
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
}
again.Written, again.Words = p.clock().UTC(), why+": laid again over the keys as they stand, to be judged and asked again"
p.seat.forget(prop.ID)
p.writeProposal(ctx, again, rev)
}
// lays is what a proposal is checked against, as the verbs check it: the modules the indexer holds, by key, and the
// nodes with a record.
func (p *passer) lays(ctx context.Context) (func(string) bool, func(string) bool) {
held := map[string]bool{}
for key := range p.w.read(delivery.ModulesBucket) {
if name, err := delivery.ParseModulesKey(key); err == nil {
held[name] = true
}
}
records, _ := worldReader{w: p.w}.Records(ctx)
return func(m string) bool { return held[m] }, func(n string) bool { _, ok := records[n]; return ok }
}
// warranted writes an approved proposal (§5): for the seed, after the export is read again and found unchanged (§7);
// each changed key by compare-and-set at the revision it was laid over; then the warrant recorded, the candidates and the
// proposal deleted, placement-changed said, and the controller kept in step (§8).
func (p *passer) warranted(ctx context.Context, prop delivery.PlacementProposal, rev uint64, akey string, current map[string]delivery.Placement, revs map[string]uint64) {
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
p.endProposal(ctx, prop, "its warrant does not bind these changes, so nothing is written: "+err.Error())
return
}
answer, _ := p.w.answer(akey)
if prop.Seed {
if p.seat == nil {
return
}
c := p.seat.exportOf(prop.ID)
switch {
case c.running:
return // its answer wakes the next pass
case c.err != "":
if words := "the export cannot be read again at the warrant: " + c.err; prop.Words != words {
prop.Words = words
p.writeProposal(ctx, prop, rev)
}
return
case delivery.ExportDigest(c.export) != prop.ExportDigest:
p.endProposal(ctx, prop, "the controller's export changed since the seed was proposed: the new export is proposed as the seed")
p.proposeSeed(ctx, prop, c.export)
return
}
}
if moved(prop, revs) {
p.layAgain(ctx, prop, rev, current, revs, "a key moved since it was approved, so its warrant is spent")
return
}
cands, _, _ := p.candidatesOf(prop.ID)
laid := map[string]*delivery.Placement{}
for _, node := range prop.Nodes() {
laid[node] = cands[node].Placement
}
writes, err := assign.Writes(prop, laid)
if err != nil {
p.logf("the proposal %s cannot be written: %v", prop.ID, err)
return
}
kv := p.kvs[delivery.PlacementBucket]
var changes []delivery.PlacementChange
writtenLaid := map[string]*delivery.Placement{}
after := maps.Clone(current)
afterRevs := maps.Clone(revs)
for _, w := range writes {
var rev2 uint64
var err error
switch {
case w.Value == nil:
err = kv.Delete(ctx, w.Key, jetstream.LastRevision(w.Base))
default:
raw, encErr := delivery.EncodePlacement(*w.Value)
if encErr != nil {
err = encErr
} else if w.Base == 0 {
rev2, err = kv.Create(ctx, w.Key, raw)
} else {
rev2, err = kv.Update(ctx, w.Key, raw, w.Base)
}
}
if err != nil {
// Another writer came first (a hand-back): what is written stands and is followed through; the rest is laid
// again over the keys as this write left them.
p.logf("placement %s is not written at revision %d (%v): the rest of the proposal %s is laid again", w.Key, w.Base, err, prop.ID)
p.followWrite(prop, answer, changes, current, writtenLaid)
if rest, ok := assign.Rest(prop, writtenLaid); ok {
p.layAgain(ctx, rest, rev, after, afterRevs, "a key moved while it was written, after "+strings.Join(sortedNodes(writtenLaid), ", ")+" were")
} else {
p.endProposal(ctx, prop, "a key moved while it was written: "+err.Error())
}
return
}
before := current[w.Node]
now := delivery.Placement{Node: w.Node}
if w.Value != nil {
now = *w.Value
after[w.Node] = *w.Value
} else {
delete(after, w.Node)
}
afterRevs[w.Node] = rev2
c := delivery.ChangeOf(before, now)
c.Before, c.After = w.Base, rev2
changes = append(changes, c)
writtenLaid[w.Node] = w.Value
}
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
p.followWrite(prop, answer, changes, current, writtenLaid)
}
// followWrite follows the keys a warrant wrote: the warrant recorded, placement-changed said, and, until the lease
// write, the controller's assignments kept in step for them (§8). The seed calls no controller verb: it is what the
// controller holds already (§9).
func (p *passer) followWrite(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange,
before map[string]delivery.Placement, written map[string]*delivery.Placement) {
if len(changes) == 0 {
return
}
p.seat.record(answer.Warrant)
p.sayChanged(prop, answer, changes)
if p.seat != nil && !p.w.handedAway() && !prop.Seed {
p.seat.callAll(assign.ControllerCalls(before, written, prop.ID, answer.Warrant))
}
}
func sortedNodes(m map[string]*delivery.Placement) []string {
out := make([]string, 0, len(m))
for n := range m {
out = append(out, n)
}
sort.Strings(out)
return out
}
// handBack deletes every key on the operator's warrant (§7) at the revision the hand-back read, records it in the
// controller's hand-act log and says placement-changed; every node returns to the controller's compositions. A key moved
// since ends it, and the operator hands back again.
func (p *passer) handBack(ctx context.Context, prop delivery.PlacementProposal, akey string, revs map[string]uint64) {
if err := p.performs(akey, assign.PlacementAct(prop)); err != nil {
p.endProposal(ctx, prop, "its warrant does not bind this hand-back, so nothing is deleted: "+err.Error())
return
}
answer, _ := p.w.answer(akey)
current, _ := p.seatPlacements()
var changes []delivery.PlacementChange
for _, node := range prop.Nodes() {
key, _ := delivery.PlacementKey(node)
if err := p.kvs[delivery.PlacementBucket].Delete(ctx, key, jetstream.LastRevision(prop.Base[node])); err != nil {
p.sayChanged(prop, answer, changes)
p.endProposal(ctx, prop, fmt.Sprintf("%s moved since the hand-back was asked (%v): hand back again", key, err))
return
}
c := delivery.ChangeOf(current[node], delivery.Placement{Node: node})
c.Before = prop.Base[node]
changes = append(changes, c)
}
if p.seat != nil {
p.seat.handAct(prop)
}
p.seat.record(answer.Warrant)
_ = p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID)
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
p.deletePulses(ctx, prop.ID)
p.sayChanged(prop, answer, changes)
}
// sayChanged says placement-changed for the keys written.
func (p *passer) sayChanged(prop delivery.PlacementProposal, answer delivery.Answer, changes []delivery.PlacementChange) {
if len(changes) == 0 {
return
}
at := p.clock().UTC()
if _, when, ok := strings.Cut(answer.Warrant, "@"); ok {
if t, err := time.Parse(time.RFC3339Nano, when); err == nil {
at = t
}
}
delivery.SortChanges(changes)
p.sayEvent(delivery.PlacementChangedEvent, delivery.PlacementEvent{Proposal: prop, Nodes: changes,
Warrant: &delivery.PlacementWarrant{Warrant: answer.Warrant, Words: answer.Words, At: at}}, prop.ID+".changed")
}
// proposeSeed proposes an export as the seed again, by the same proposer and why.
func (p *passer) proposeSeed(ctx context.Context, was delivery.PlacementProposal, export map[string]delivery.Placement) {
held, recorded := p.lays(ctx)
now := p.clock()
seed, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: was.By, Why: was.Why, At: now,
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: p.w.handedAway()})
if len(findings) > 0 {
p.logf("the new export is not proposed as the seed: %s", strings.Join(findings, "; "))
return
}
for _, node := range seed.Nodes() {
key, _ := delivery.PlacementCandidateKey(seed.ID, node)
c := delivery.PlacementCandidate{Proposal: seed.ID, Placement: laid[node]}
p.put(ctx, delivery.PlacementCandidatesBucket, key, func() ([]byte, error) { return delivery.EncodePlacementCandidate(c) })
}
seed.Written = now.UTC()
p.writeProposal(ctx, seed, 0)
}
// endProposal ends a proposal: its ask, its candidates and its key deleted, and placement-withdrawn said carrying it
// whole with why it ended.
func (p *passer) endProposal(ctx context.Context, prop delivery.PlacementProposal, why string) {
if akey, err := delivery.PlacementAskKey(prop.ID); err == nil && p.w.has(delivery.AssignerAsksBucket, akey) {
if err := p.kvs[delivery.AssignerAsksBucket].Delete(ctx, akey); err == nil {
p.w.put(delivery.AssignerAsksBucket, akey, nil)
}
}
p.deleteCandidates(ctx, prop.ID)
p.deletePulses(ctx, prop.ID)
if err := p.kvs[delivery.PlacementProposalsBucket].Delete(ctx, prop.ID); err != nil && !errors.Is(err, context.Canceled) {
p.logf("the proposal %s is not deleted, and is ended at the next pass: %v", prop.ID, err)
return
}
p.w.put(delivery.PlacementProposalsBucket, prop.ID, nil)
p.seat.forget(prop.ID)
p.logf("the proposal %s ends: %s", prop.ID, why)
p.sayEvent(delivery.PlacementWithdrawnEvent, delivery.PlacementEvent{Proposal: prop, Ended: why}, prop.ID+".withdrawn")
}
// deleteCandidates deletes every candidate of a proposal.
func (p *passer) deleteCandidates(ctx context.Context, id string) {
for key := range p.w.read(delivery.PlacementCandidatesBucket) {
if pid, _, err := delivery.ParsePlacementCandidateKey(key); err == nil && pid == id {
if err := p.kvs[delivery.PlacementCandidatesBucket].Delete(ctx, key); err == nil {
p.w.put(delivery.PlacementCandidatesBucket, key, nil)
}
}
}
}
// writeProposal writes a proposal over the revision the pass read, by compare-and-set: a verb that marked it meanwhile
// (withdraw, again) wins, and the next pass reads that. It says whether it wrote.
func (p *passer) writeProposal(ctx context.Context, prop delivery.PlacementProposal, rev uint64) bool {
raw, err := delivery.EncodePlacementProposal(prop)
if err != nil {
p.logf("the proposal %s is not written: %v", prop.ID, err)
return false
}
kv := p.kvs[delivery.PlacementProposalsBucket]
var at uint64
if rev == 0 {
at, err = kv.Create(ctx, prop.ID, raw)
} else {
at, err = kv.Update(ctx, prop.ID, raw, rev)
}
if err != nil {
if !errors.Is(err, context.Canceled) {
p.logf("the proposal %s is not written, and is taken further at the next pass: %v", prop.ID, err)
}
return false
}
p.w.putEntry(delivery.PlacementProposalsBucket, prop.ID, raw, at, p.clock())
return true
}
// say says one of the seat's events, said in the journal when the bus refuses it.
func (p *passer) sayEvent(event string, body delivery.PlacementEvent, id string) {
if p.publish == nil {
return
}
if err := p.publish(event, body, id); err != nil {
p.logf("%s of %s is not said: %v", event, body.Proposal.ID, err)
}
}
@@ -1,543 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
)
// The seat mesh-placement's seven verbs (novox/hq ADR 0567 §10), served through the tool runner beside the put tools: a
// process never serves (ADR 0436). They read every state the seat keeps and write what a person asks of it: a proposal
// and its candidates (propose, adopt), a proposal marked to end (withdraw) or to be judged again (again), and the keys
// deleted (hand-back). Everything that follows a proposal — composing it, judging the answers, the ask, the write at the
// warrant and the seat's events on its stream — is the assigner's process's (seat.go), started by these writes.
// seatStore is what the seat's verbs need of the states: the runtime's, or a test's.
type seatStore interface {
store
del(state, key string) error
history(state, key string) ([]stdio.StateRevision, error)
}
func (runtimeStore) del(state, key string) error { return stdio.State(state).Delete(key) }
func (runtimeStore) history(state, key string) ([]stdio.StateRevision, error) {
return stdio.State(state).History(key)
}
// The states the verbs read, as the runtime names them: the seat's own by their local names, the others'
// `<module>.<state>`.
const (
modulesState = delivery.IndexerModule + ".modules"
haveState = "node-engine.have"
)
// seatTools serves the seat's verbs over a store, asking the controller through ask (stdio.Ask, or a test's).
type seatTools struct {
st seatStore
ask func(key string, body any) (json.RawMessage, error)
now func() time.Time
}
func newSeatTools(st seatStore, ask func(string, any) (json.RawMessage, error)) *seatTools {
return &seatTools{st: st, ask: ask, now: time.Now}
}
// keyed is one placement key as a verb answers it.
type keyed struct {
Placement delivery.Placement `json:"placement"`
Revision uint64 `json:"revision"`
}
// placements reads every node's placement and its key's revision.
func (s *seatTools) placements() (map[string]delivery.Placement, map[string]uint64, error) {
keys, err := s.st.keys(delivery.PlacementState)
if err != nil {
return nil, nil, fmt.Errorf("placement cannot be listed: %w", err)
}
out, revs := map[string]delivery.Placement{}, map[string]uint64{}
for _, key := range keys {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
continue
}
raw, rev, found, err := s.st.get(delivery.PlacementState, key)
if err != nil {
return nil, nil, fmt.Errorf("placement of %s cannot be read: %w", node, err)
}
if !found {
continue
}
p, err := delivery.DecodePlacement(raw)
if err != nil {
return nil, nil, fmt.Errorf("placement of %s cannot be read: %w", node, err)
}
out[node], revs[node] = p, rev
}
return out, revs, nil
}
// lays is what a proposal is checked against: the modules the indexer holds and the nodes with a record.
func (s *seatTools) lays() (func(string) bool, func(string) bool, error) {
mods, err := s.st.keys(modulesState)
if err != nil {
return nil, nil, fmt.Errorf("the indexer's modules cannot be listed: %w", err)
}
held := map[string]bool{}
for _, k := range mods {
if name, err := delivery.ParseModulesKey(k); err == nil {
held[name] = true
}
}
haves, err := s.st.keys(haveState)
if err != nil {
return nil, nil, fmt.Errorf("the nodes' records cannot be listed: %w", err)
}
recorded := map[string]bool{}
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
if raw, _, found, err := s.st.get(haveState, k); err == nil && found {
if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil {
recorded[node] = true
}
}
}
return func(m string) bool { return held[m] }, func(n string) bool { return recorded[n] }, nil
}
// proposal reads one open proposal.
func (s *seatTools) proposal(id string) (delivery.PlacementProposal, error) {
if err := delivery.CheckProposalID(id); err != nil {
return delivery.PlacementProposal{}, err
}
raw, _, found, err := s.st.get(delivery.PlacementProposalsState, id)
if err != nil {
return delivery.PlacementProposal{}, fmt.Errorf("the proposal %s cannot be read: %w", id, err)
}
if !found {
return delivery.PlacementProposal{}, fmt.Errorf("no proposal %s is open: mesh-placement.proposals lists those that are", id)
}
return delivery.DecodePlacementProposal(raw)
}
// write writes a proposal laid by assign.Propose: each candidate, then the proposal, whose write the assigner's process
// takes as the event to compose and judge it.
func (s *seatTools) write(p delivery.PlacementProposal, laid map[string]*delivery.Placement) error {
p.Written = s.now().UTC()
for _, node := range p.Nodes() {
key, err := delivery.PlacementCandidateKey(p.ID, node)
if err != nil {
return err
}
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: p.ID, Placement: laid[node]})
if err != nil {
return err
}
if err := s.st.put(delivery.PlacementCandidatesState, key, raw); err != nil {
return fmt.Errorf("the candidate %s is not written: %w", key, err)
}
}
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return fmt.Errorf("the proposal %s is not written: %w", p.ID, err)
}
return nil
}
// proposed is what propose and adopt answer.
type proposed struct {
Proposal delivery.PlacementProposal `json:"proposal"`
Follow string `json:"follow"`
}
func followProposal(id string) string {
return fmt.Sprintf("the nodes it touches answer it; mesh-placement.proposals with id %q says where it stands, and the "+
"operator is asked to approve it once its verdict passes", id)
}
// changesOf reads propose's changes: a JSON array, given as one or as its text.
func changesOf(v any) ([]delivery.PlacementProposalChange, error) {
var raw []byte
switch c := v.(type) {
case string:
raw = []byte(c)
case nil:
return nil, errors.New("propose names its changes")
default:
var err error
if raw, err = json.Marshal(c); err != nil {
return nil, err
}
}
var out []delivery.PlacementProposalChange
dec := json.NewDecoder(strings.NewReader(string(raw)))
dec.DisallowUnknownFields()
if err := dec.Decode(&out); err != nil {
return nil, fmt.Errorf("the changes are no list of changes: %w", err)
}
return out, nil
}
// doPropose is the verb propose (§2).
func (s *seatTools) doPropose(args map[string]any, by string) (any, error) {
changes, err := changesOf(args["changes"])
if err != nil {
return nil, err
}
for _, c := range changes {
if c.Op == delivery.OpSeed {
return nil, errors.New("the seed is adopt's alone: propose takes place, take-off, pin, unpin, hold, unhold, decide and config")
}
}
current, revs, err := s.placements()
if err != nil {
return nil, err
}
held, recorded, err := s.lays()
if err != nil {
return nil, err
}
now := s.now()
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: changes, Current: current, Revisions: revs, Held: held, Recorded: recorded, Leased: s.leased()})
if len(findings) > 0 {
return nil, fmt.Errorf("the proposal is refused: %s", strings.Join(findings, "; "))
}
if err := s.write(p, laid); err != nil {
return nil, err
}
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
}
// doWithdraw is the verb withdraw (§2): the proposal is marked, and the assigner deletes it and its candidates and says
// placement-withdrawn carrying it whole.
func (s *seatTools) doWithdraw(args map[string]any, by string) (any, error) {
why := arg(args, "why")
if why == "" {
return nil, errors.New("withdraw says why: it is kept in the event that ends the proposal")
}
p, err := s.proposal(arg(args, "id"))
if err != nil {
return nil, err
}
p.Withdrawn = fmt.Sprintf("withdrawn by %s: %s", by, why)
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not withdrawn: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "withdrawn": p.Withdrawn}, nil
}
// doAgain is the verb again (§10): the proposal is marked, and the assigner judges it now and raises its ask again
// when its verdict passes.
func (s *seatTools) doAgain(args map[string]any) (any, error) {
p, err := s.proposal(arg(args, "id"))
if err != nil {
return nil, err
}
p.Again = s.now().UTC()
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the proposal %s is not asked again: %w", p.ID, err)
}
return map[string]any{"proposal": p.ID, "again": p.Again, "follow": followProposal(p.ID)}, nil
}
// exportCall is the controller's verb the seed is read from, by request and reply on its seat (§7).
const exportCall = "seat:" + catalogue.ControllerSeatName + ".placement-export"
// doAdopt is the verb adopt (§7): the controller's export proposed whole as the seed.
func (s *seatTools) doAdopt(args map[string]any, by string) (any, error) {
current, _, err := s.placements()
if err != nil {
return nil, err
}
if len(current) > 0 {
return nil, fmt.Errorf("placement is taken over already (%d keys): propose a change instead, or hand it back first", len(current))
}
if ids, err := s.st.keys(delivery.PlacementProposalsState); err == nil {
for _, id := range ids {
if p, err := s.proposal(id); err == nil && p.Seed {
return nil, fmt.Errorf("the seed %s is open: withdraw it first, or answer its ask", id)
}
}
}
raw, err := s.ask(exportCall, map[string]any{})
if err != nil {
return nil, fmt.Errorf("the controller's placement-export did not answer: %w", err)
}
export, err := assign.ExportOf(raw)
if err != nil {
return nil, err
}
held, recorded, err := s.lays()
if err != nil {
return nil, err
}
now := s.now()
p, laid, findings := assign.Propose(assign.ProposeIn{ID: delivery.NewProposalID(now), By: by, Why: arg(args, "why"), At: now,
Changes: assign.SeedChanges(export), Current: map[string]delivery.Placement{}, Revisions: map[string]uint64{},
Held: held, Recorded: recorded, Seed: true, ExportDigest: delivery.ExportDigest(export), Leased: s.leased()})
if len(findings) > 0 {
return nil, fmt.Errorf("the seed is refused: %s", strings.Join(findings, "; "))
}
if err := s.write(p, laid); err != nil {
return nil, err
}
return proposed{Proposal: p, Follow: followProposal(p.ID)}, nil
}
// leased says the controller's lease is handed away (mesh-sdk buscompose.LeaseHandedAway, ADR 0561 §3; issue 774): the lease
// write of ADR 0547 that ends keeping the controller in step (ADR 0567 §8). A lease not read, a controller's, expired or
// unreadable, is not handed away.
func (s *seatTools) leased() bool {
raw, _, found, err := s.st.get(buscompose.ControllerLease, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, err)
return away
}
// handActCall records a hand act in the controller's hand-act log.
const handActCall = "seat:" + catalogue.ControllerSeatName + ".hand-act"
// doHandBack is the verb hand-back (§7): the operator's alone, so it is asked as a proposal is. It writes a hand-back
// proposal naming every key at its revision; the assigner raises its authorising ask and, on its warrant, deletes the keys,
// records the hand act and says placement-changed; every node returns to the controller's compositions.
func (s *seatTools) doHandBack(args map[string]any, by string) (any, error) {
if s.leased() {
return nil, errors.New("placement is the mesh's alone since the lease write: there is no controller to hand it back to")
}
why := arg(args, "why")
if why == "" {
return nil, errors.New("hand-back says why: it is recorded with the hand act")
}
if by == "" {
return nil, errors.New("the runtime names no caller: a hand-back is made by someone")
}
_, revs, err := s.placements()
if err != nil {
return nil, err
}
if len(revs) == 0 {
return nil, errors.New("there is no placement key to hand back")
}
now := s.now()
p := delivery.PlacementProposal{ID: delivery.NewProposalID(now), By: by, Why: why, At: now.UTC(), Written: now.UTC(),
HandBack: true, Base: revs, State: delivery.ProposalAnswering}
raw, err := delivery.EncodePlacementProposal(p)
if err != nil {
return nil, err
}
if err := s.st.put(delivery.PlacementProposalsState, p.ID, raw); err != nil {
return nil, fmt.Errorf("the hand-back is not written: %w", err)
}
return proposed{Proposal: p, Follow: "the operator is asked to approve it; on the warrant every key is deleted and each " +
"node returns to the controller's compositions"}, nil
}
// placementAnswer is what the verb placement answers.
type placementAnswer struct {
Keys map[string]keyed `json:"keys,omitempty"`
Nodes []string `json:"nodes,omitempty"`
Count int `json:"count"`
History int `json:"history-per-key"`
Changes []historyEntry `json:"changes,omitempty"`
Older string `json:"older,omitempty"`
}
// historyEntry is one change of a key in the state's revisions.
type historyEntry struct {
Node string `json:"node"`
Revision uint64 `json:"revision"`
At time.Time `json:"at"`
Change delivery.PlacementChange `json:"change"`
Deleted bool `json:"deleted,omitempty"`
}
// doPlacement is the verb placement (§10, §6): one node's key, one module's nodes, or every key; with history, the
// changes in order from the revisions the state keeps.
func (s *seatTools) doPlacement(args map[string]any) (any, error) {
all, revs, err := s.placements()
if err != nil {
return nil, err
}
node, module := arg(args, "node"), arg(args, "module")
out := placementAnswer{Keys: map[string]keyed{}, Count: len(all), History: delivery.PlacementHistory}
for n, p := range all {
if node != "" && n != node {
continue
}
if module != "" && !strings.Contains(" "+strings.Join(p.Assigned(), " ")+" ", " "+module+" ") {
continue
}
out.Keys[n] = keyed{Placement: p, Revision: revs[n]}
out.Nodes = append(out.Nodes, n)
}
sort.Strings(out.Nodes)
if node != "" && len(out.Keys) == 0 && arg(args, "history") != "true" {
return nil, fmt.Errorf("%s has no placement key", node)
}
if arg(args, "history") != "true" {
return out, nil
}
nodes := out.Nodes
if node != "" {
nodes = []string{node}
} else if module == "" {
nodes = nil
for n := range all {
nodes = append(nodes, n)
}
sort.Strings(nodes)
}
for _, n := range nodes {
key, err := delivery.PlacementKey(n)
if err != nil {
return nil, err
}
revisions, err := s.st.history(delivery.PlacementState, key)
if err != nil {
return nil, fmt.Errorf("the history of %s cannot be read: %w", key, err)
}
var before delivery.Placement
for _, r := range revisions {
e := historyEntry{Node: n, Revision: r.Revision, At: r.At}
after := delivery.Placement{}
if r.Op == "" || r.Op == "put" || r.Op == "PUT" {
if p, err := delivery.DecodePlacement(r.Value); err == nil {
after = p
}
} else {
e.Deleted = true
}
e.Change = delivery.ChangeOf(before, after)
e.Change.Before, e.Change.After = 0, r.Revision
before = after
if module != "" && !strings.Contains(" "+strings.Join(append(append(e.Change.Added, e.Change.Removed...), e.Change.Changed...), " ")+" ", " "+module+" ") {
continue
}
out.Changes = append(out.Changes, e)
}
}
out.Older = fmt.Sprintf("the state keeps %d revisions of each key; an older change is in its placement-changed event on "+
"the assigner's stream %s, kept %d days", delivery.PlacementHistory, delivery.PlacementStream, delivery.PlacementStreamDays)
return out, nil
}
// openProposal is one open proposal as the verb proposals answers it: the proposal (its changes, base revisions,
// verdict with the differences and the answers it judged, its ask), its candidates by node with their revisions, and
// every node's answer now.
type openProposal struct {
Proposal delivery.PlacementProposal `json:"proposal"`
Candidates map[string]uint64 `json:"candidates"`
Answers map[string]delivery.CandidateAnswer `json:"answers,omitempty"`
}
// doProposals is the verb proposals (§10).
func (s *seatTools) doProposals(args map[string]any) (any, error) {
ids, err := s.st.keys(delivery.PlacementProposalsState)
if err != nil {
return nil, fmt.Errorf("the proposals cannot be listed: %w", err)
}
cands, err := s.st.keys(delivery.PlacementCandidatesState)
if err != nil {
return nil, fmt.Errorf("the candidates cannot be listed: %w", err)
}
haves, err := s.st.keys(haveState)
if err != nil {
return nil, fmt.Errorf("the nodes' haves cannot be listed: %w", err)
}
answers := map[string]map[string]delivery.CandidateAnswer{} // by proposal, then node
for _, k := range haves {
node, _, isNode, err := delivery.ParseHaveKey(k)
if err != nil || !isNode {
continue
}
raw, _, found, err := s.st.get(haveState, k)
if err != nil || !found {
continue
}
h, err := delivery.DecodeHaveNode(raw)
if err != nil {
continue
}
for id, a := range h.Candidates {
if answers[id] == nil {
answers[id] = map[string]delivery.CandidateAnswer{}
}
answers[id][node] = a
}
}
sort.Strings(ids)
out := []openProposal{}
for _, id := range ids {
if want := arg(args, "id"); want != "" && want != id {
continue
}
p, err := s.proposal(id)
if err != nil {
return nil, err
}
o := openProposal{Proposal: p, Candidates: map[string]uint64{}, Answers: answers[id]}
for _, k := range cands {
pid, node, err := delivery.ParsePlacementCandidateKey(k)
if err != nil || pid != id {
continue
}
if _, rev, found, err := s.st.get(delivery.PlacementCandidatesState, k); err == nil && found {
o.Candidates[node] = rev
}
}
out = append(out, o)
}
if arg(args, "id") != "" && len(out) == 0 {
return nil, fmt.Errorf("no proposal %s is open", arg(args, "id"))
}
return map[string]any{"proposals": out}, nil
}
// tools are the seat's verbs as the runtime serves them, each named <seat>.<verb> (the tool runner serves a seat's verb
// only from a tool of that name), with the seat's own schemas.
func (s *seatTools) tools() []stdio.Tool {
seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat)
input := map[string]map[string]any{}
description := map[string]string{}
for _, v := range seat.Serves {
input[v.Name], description[v.Name] = v.Input, v.Description
}
run := map[string]func(map[string]any) (any, error){
catalogue.PlacementVerbPlacement: s.doPlacement,
catalogue.PlacementVerbProposals: s.doProposals,
catalogue.PlacementVerbPropose: func(a map[string]any) (any, error) { return s.doPropose(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbWithdraw: func(a map[string]any) (any, error) { return s.doWithdraw(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbAgain: s.doAgain,
catalogue.PlacementVerbAdopt: func(a map[string]any) (any, error) { return s.doAdopt(a, stdio.CallerOf(a)) },
catalogue.PlacementVerbHandBack: func(a map[string]any) (any, error) { return s.doHandBack(a, stdio.CallerOf(a)) },
}
var out []stdio.Tool
for _, v := range seat.Serves {
out = append(out, stdio.Tool{Name: catalogue.PlacementSeat + "." + v.Name, Description: description[v.Name], Input: input[v.Name], Run: run[v.Name]})
}
return out
}
@@ -1,167 +0,0 @@
package main
import (
"encoding/json"
"os"
"slices"
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func (s memStore) del(state, key string) error {
delete(s[state].values, key)
return nil
}
func (s memStore) history(state, key string) ([]stdio.StateRevision, error) {
v, ok := s[state].values[key]
if !ok {
return nil, nil
}
return []stdio.StateRevision{{Key: key, Value: v, Revision: uint64(len(v)), Op: "put"}}, nil
}
// ADR 0567 §2, §7, §10: the seat's seven verbs over the states. propose lays a proposal and writes it with its candidates,
// or refuses naming every finding and writes nothing; withdraw and again mark it for the holder; adopt proposes the
// controller's export whole as the seed, once; hand-back records the hand act and deletes every key; placement and
// proposals read every state the seat keeps, and history names the stream for what the state no longer keeps.
func TestTheSeatsVerbs(t *testing.T) {
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Modules: []delivery.PlacedModule{{Module: "web"}}}
st := memStore{}
for _, state := range []string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState, modulesState, haveState} {
st[state] = &memKV{values: map[string][]byte{}}
}
for _, m := range []string{"web", "shell"} {
st[modulesState].values[m] = []byte("{}")
}
for _, n := range []string{"anchor", "laptop"} {
raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: n}})
st[haveState].values[n+".node"] = raw
}
export, _ := json.Marshal(map[string]any{"anchor": anchor})
var asked []string
s := newSeatTools(st, func(key string, body any) (json.RawMessage, error) {
asked = append(asked, key)
if key == exportCall {
out, _ := json.Marshal(map[string]any{"ok": true, "output": string(export)})
return out, nil
}
return json.RawMessage(`{"ok":true}`), nil
})
s.now = func() time.Time { return time.Date(2026, 10, 28, 20, 0, 0, 0, time.UTC) }
// adopt: the export proposed whole as the seed, once.
got, err := s.doAdopt(map[string]any{"why": "take over"}, "operator@laptop")
if err != nil {
t.Fatal(err)
}
seed := got.(proposed).Proposal
if !seed.Seed || seed.ExportDigest != delivery.ExportDigest(map[string]delivery.Placement{"anchor": anchor}) || !slices.Equal(asked, []string{exportCall}) ||
len(st[delivery.PlacementCandidatesState].values) != 1 || st[delivery.PlacementProposalsState].values[seed.ID] == nil {
t.Fatalf("adopt = %+v, asked %v", seed, asked)
}
if _, err := s.doAdopt(map[string]any{"why": "again"}, "op"); err == nil || !strings.Contains(err.Error(), "is open") {
t.Errorf("a second adopt with a seed open: %v", err)
}
// withdraw and again mark the proposal; the holder ends or judges it.
if _, err := s.doWithdraw(map[string]any{"id": seed.ID, "why": "not yet"}, "op"); err != nil {
t.Fatal(err)
}
if _, err := s.doAgain(map[string]any{"id": seed.ID}); err != nil {
t.Fatal(err)
}
if p, err := s.proposal(seed.ID); err != nil || !strings.Contains(p.Withdrawn, "not yet") || p.Again.IsZero() {
t.Errorf("marked = %+v, %v", p, err)
}
if _, err := s.doWithdraw(map[string]any{"id": "p-none", "why": "x"}, "op"); err == nil {
t.Error("an unknown proposal was withdrawn")
}
delete(st[delivery.PlacementProposalsState].values, seed.ID) // the holder ended it
st[delivery.PlacementCandidatesState].values = map[string][]byte{}
// propose: refused whole, naming every finding; a valid move laid over the keys at their revisions.
raw, _ := delivery.EncodePlacement(anchor)
st[delivery.PlacementState].values["placement.anchor"] = raw
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"place","node":"anchor","module":"ghost"},{"op":"hold","node":"anchor","module":"web","claim":"x"}]`}, "op"); err == nil ||
!strings.Contains(err.Error(), "ghost") || !strings.Contains(err.Error(), "lease write") || len(st[delivery.PlacementProposalsState].values) != 0 {
t.Errorf("a refused proposal: %v", err)
}
if _, err := s.doPropose(map[string]any{"why": "x", "changes": `[{"op":"swap","node":"anchor"}]`}, "op"); err == nil {
t.Error("an unknown change was taken")
}
got, err = s.doPropose(map[string]any{"why": "move web", "changes": []any{
map[string]any{"op": "take-off", "node": "anchor", "module": "web"}, map[string]any{"op": "place", "node": "laptop", "module": "web"}}}, "operator@laptop")
if err != nil {
t.Fatal(err)
}
move := got.(proposed).Proposal
if move.By != "operator@laptop" || move.Base["anchor"] != uint64(len(raw)) || move.Base["laptop"] != 0 || len(st[delivery.PlacementCandidatesState].values) != 2 {
t.Fatalf("propose = %+v", move)
}
// placement and proposals read every state the seat keeps (§10); history names the stream for what is older.
reads := map[string]bool{}
pa, err := s.doPlacement(map[string]any{"module": "web", "history": "true"})
if a := pa.(placementAnswer); err != nil || !slices.Equal(a.Nodes, []string{"anchor"}) || len(a.Changes) != 1 || !strings.Contains(a.Older, delivery.PlacementStream) {
t.Errorf("placement of web = %+v, %v", pa, err)
} else {
reads[delivery.PlacementState], reads["stream "+delivery.PlacementStream] = true, true
}
if _, err := s.doPlacement(map[string]any{"node": "laptop"}); err == nil {
t.Error("a node with no key was answered")
}
ps, err := s.doProposals(map[string]any{"id": move.ID})
if list := ps.(map[string]any)["proposals"].([]openProposal); err != nil || len(list) != 1 || len(list[0].Candidates) != 2 {
t.Errorf("proposals = %+v, %v", ps, err)
} else {
reads[delivery.PlacementProposalsState], reads[delivery.PlacementCandidatesState] = true, true
}
raw0, err := readManifest()
if err != nil {
t.Fatal(err)
}
for _, state := range raw0.State {
if slices.Contains([]string{delivery.PlacementState, delivery.PlacementProposalsState, delivery.PlacementCandidatesState}, state.Name) && !reads[state.Name] {
t.Errorf("no verb reads the seat's state %s", state.Name)
}
}
for _, stream := range raw0.Streams {
if !reads["stream "+stream.Name] {
t.Errorf("no verb reads the seat's stream %s", stream.Name)
}
}
// hand-back: the operator's alone, written as a hand-back proposal naming every key at its revision, for the assigner
// to ask; nothing is deleted by the verb.
got, err = s.doHandBack(map[string]any{"why": "back"}, "op")
if hb := got.(proposed).Proposal; err != nil || !hb.HandBack || hb.Base["anchor"] != uint64(len(raw)) || len(st[delivery.PlacementState].values) != 1 {
t.Errorf("hand-back = %+v, %v", got, err)
}
// Issue 774 (ADR 0567 §8): once the controller's lease is handed away a change its verbs cannot take is proposed, and
// there is no controller to hand placement back to.
st[buscompose.ControllerLease] = &memKV{values: map[string][]byte{buscompose.ControllerLeaseKey: []byte(`{"handed-to":"the operator's hand-over","handed-by":"person.jochen"}`)}}
if _, err := s.doPropose(map[string]any{"why": "hold", "changes": `[{"op":"hold","node":"anchor","module":"web","claim":"node-shell","scope":"node"}]`}, "op"); err != nil {
t.Errorf("a hold after the lease write: %v", err)
}
if _, err := s.doHandBack(map[string]any{"why": "back"}, "op"); err == nil || !strings.Contains(err.Error(), "since the lease write") {
t.Errorf("a hand-back after the lease write: %v", err)
}
if tools := s.tools(); len(tools) != 7 || tools[2].Name != "mesh-placement.propose" || tools[2].Run == nil || tools[2].Input == nil {
t.Errorf("the seat's verbs as served: %+v", tools)
}
}
// readManifest is the module's manifest, read as the controller reads it.
func readManifest() (catalogue.Manifest, error) {
raw, err := os.ReadFile("../../module.json")
if err != nil {
return catalogue.Manifest{}, err
}
return catalogue.ParseManifest(raw)
}
@@ -18,8 +18,8 @@ import (
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
// 0436, mesh-sdk process). They write and read the puts; the assigner's process takes each put further (puts.go) and says
// every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them
// (seat_tools.go, ADR 0567 §10).
// every step as the event put. They are the module's own tools, not a seat's verbs: a new seat needs a change of the
// controller, which takes none; they become the verbs of a seat once composing leaves the controller.
// putsState is the assigner's state of puts, as the runtime names a module's own state.
const putsState = "puts"
@@ -351,7 +351,7 @@ func serveTools() error {
time.Sleep(wait)
}
}()
return stdio.Serve("", append(s.tools(), newSeatTools(runtimeStore{}, stdio.Ask).tools()...))
return stdio.Serve("", s.tools())
}
// shortFP is a fingerprint's first twelve hex digits.
@@ -13,7 +13,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
@@ -193,41 +192,24 @@ func (w *world) manifest(ctx context.Context, get getter, digest string) parsed
return p
}
// assigned is each node's modules both the controller's assignments and the seat mesh-placement's placement name, in name order
// assigned is each node's modules both the controller's assignments and the indexer's placement name, in name order
// (ADR 0549 §5; the assignments alone before any placement was read), with every open assignment by node and module.
// Once the controller's lease is handed away it is the seat's placement alone, with no assignment (ADR 0549 §5, §6).
func (w *world) assigned() (map[string][]string, map[string]assignment) {
// A lease handed away before any placement was read keeps the assignments as they last stood, never an empty mesh:
// the lease write's own check refuses that order (ADR 0547 §11).
placed, seeded := w.placed()
byAssignment, all := w.assignedByController()
if away := w.handedAway(); away && seeded {
both, _ := assign.Placed(byAssignment, placed, seeded, away)
return both, map[string]assignment{}
}
both, _ := assign.Placed(byAssignment, placed, seeded, false)
placed, seeded := w.placed()
both, _ := assign.Placed(byAssignment, placed, seeded)
return both, all
}
// placementDiffers is every pair the controller's assignments and the seat's placement do not both name: none once
// the controller's lease is handed away, since the placement is then the one source.
// placementDiffers is every pair the controller's assignments and the indexer's placement do not both name.
func (w *world) placementDiffers() []assign.Pair {
placed, seeded := w.placed()
byAssignment, _ := w.assignedByController()
_, differ := assign.Placed(byAssignment, placed, seeded, w.handedAway())
placed, seeded := w.placed()
_, differ := assign.Placed(byAssignment, placed, seeded)
return differ
}
// handedAway says the controller's lease is handed to a holder that is no controller (mesh-sdk buscompose.LeaseHandedAway,
// ADR 0561 §3): from then the seat mesh-placement's placement alone names where modules run (ADR 0549 §5). A lease not read (its
// bucket not granted or not there), a controller's, expired or unreadable, is not handed away.
func (w *world) handedAway() bool {
raw, found := w.read(buscompose.LeaseBucket)[buscompose.ControllerLeaseKey]
away, _ := buscompose.LeaseHandedAway(raw, found && raw != nil, nil)
return away
}
// placed is each node's modules in the seat's placement (ADR 0567 §1), and whether any placement key was read: a key
// placed is each node's modules in the indexer's placement (ADR 0549 §2), and whether any placement key was read: a key
// that does not read places nothing on its node and is said by the indexer's own read tool.
func (w *world) placed() (map[string][]string, bool) {
out := map[string][]string{}
@@ -246,7 +228,7 @@ func (w *world) placed() (map[string][]string, bool) {
return out, len(values) > 0
}
// placedNodes is every node the seat's placement names (ADR 0567 §1): those compose their own modules (ADR 0547 §2),
// placedNodes is every node the indexer's placement names (ADR 0549 §2): those compose their own modules (ADR 0547 §2),
// so their wants name the build alone and the seventh rule does not hold them.
func (w *world) placedNodes() map[string]bool {
placed, _ := w.placed()
@@ -320,7 +302,7 @@ func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
return m
}
// worldReader is what the assigner watched as frombus reads the bus: the seat's placement, the indexer's modules, each node's
// worldReader is what the assigner watched as frombus reads the bus: the indexer's placement and modules, each node's
// record in its own have key, the manifests by digest; the check reads no config value, own secret or credential (the
// composition it asks resolves, and stands in for what it does not carry).
type worldReader struct {
@@ -376,25 +358,7 @@ func (r worldReader) Manifest(ctx context.Context, digest string) ([]byte, error
return []byte(m.JSON), nil
}
// Values is one module's config values on one node as mesh-config resolved them, each a value its source lays
// (catalogue.LaidValue): the holder composes every node a proposal touches with them (ADR 0567 §9).
func (r worldReader) Values(_ context.Context, node, module string) (map[string]any, error) {
var out map[string]any
prefix := node + "." + module + "."
for key, raw := range r.w.read(buscompose.ResolvedBucket) {
name, ok := strings.CutPrefix(key, prefix)
if !ok || strings.Contains(name, ".") {
continue
}
if v, laid := catalogue.LaidValue(raw); laid {
if out == nil {
out = map[string]any{}
}
out[name] = v
}
}
return out, nil
}
func (worldReader) Values(context.Context, string, string) (map[string]any, error) { return nil, nil }
func (worldReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
return nil, nil
@@ -404,8 +368,8 @@ func (worldReader) Credentials(context.Context, string, []compose.CredentialKey,
return nil, nil, nil
}
// Build and Archive are read only for a node composing itself (frombus.Scope without Every): the check of every node
// composes from the manifests the indexer read, and reads no build and no archive.
// Build, Archive and Served are read only for a node composing itself (frombus.Scope without Every): the check of every
// node composes from the manifests the indexer read, reads no build and no archive, and reads no config value.
func (worldReader) Build(context.Context, string, string) (*delivery.BuilderBuild, error) {
return nil, nil
}
@@ -414,18 +378,8 @@ func (worldReader) Archive(_ context.Context, url, _ string) ([]byte, error) {
return nil, fmt.Errorf("the composition check reads no archive (%s)", url)
}
func (r worldReader) Served(ctx context.Context, node, module string, keys []string) (map[string]any, error) {
all, err := r.Values(ctx, node, module)
var out map[string]any
for _, k := range keys {
if v, ok := all[k]; ok {
if out == nil {
out = map[string]any{}
}
out[k] = v
}
}
return out, err
func (worldReader) Served(context.Context, string, string, []string) (map[string]any, error) {
return nil, nil
}
// change is a candidate's change for the composition check, or why the composition waits for it.
@@ -477,9 +431,77 @@ func (w *world) change(ctx context.Context, manifests getter, c delivery.Candida
sort.Strings(ch.Twice[name])
}
}
cp, waits := w.placementOf(ctx, manifests, c)
if waits != "" {
return ch, waits
}
ch.Placement = cp
return ch, ""
}
// placementOf is the change's own placement (ADR 0559 §5): the indexer's placement candidates of the group's catalogue
// pull request, with the input the nodes compose from with them laid over the trunk's placement. None when the group has
// no catalogue pull request or it changes no placement file. It waits while a key names another head than the group's.
func (w *world) placementOf(ctx context.Context, manifests getter, c delivery.Candidate) (*assign.ChangePlacement, string) {
var pull *delivery.CandidatePull
for i := range c.Pulls {
if delivery.SameRepository(c.Pulls[i].Repository, delivery.CatalogueRepository) {
pull = &c.Pulls[i]
}
}
if pull == nil {
return nil, ""
}
cp := &assign.ChangePlacement{Placements: map[string]*delivery.Placement{}, Revisions: map[string]uint64{}, Head: pull.Head.Head}
for key, value := range w.read(delivery.PlacementCandidatesBucket) {
group, node, err := delivery.ParsePlacementCandidateKey(key)
if err != nil || group != c.Group {
continue
}
pc, err := delivery.DecodePlacementCandidate(value)
if err != nil || pc.Pull.Number != pull.Number || !delivery.SameRepository(pc.Pull.Repository, pull.Repository) {
continue
}
if pc.Pull.Head != pull.Head.Head {
return nil, fmt.Sprintf("the indexer holds the placement candidates of %s#%d at %s, not yet at its head %s", pull.Repository, pull.Number, pc.Pull.Head, pull.Head.Head)
}
meta := w.metaOf(delivery.PlacementCandidatesBucket, key)
cp.Placements[node], cp.Revisions[node] = pc.Placement, meta.revision
if meta.at.After(cp.Written) {
cp.Written = meta.at
}
}
if len(cp.Placements) == 0 {
return nil, ""
}
in, err := frombus.Input(ctx, worldReader{w: w, manifests: manifests}, frombus.Mesh{}, frombus.Scope{Every: true, Placements: cp.Placements})
if err != nil {
cp.Unread = err.Error()
} else {
cp.Input = &in
}
return cp, ""
}
// candidateAnswers is, by node, each node's answer to the group's placement candidates in its `<node>.node` have.
func (w *world) candidateAnswers(group string) map[string]*delivery.CandidateAnswer {
out := map[string]*delivery.CandidateAnswer{}
for key, value := range w.read(delivery.HaveBucket) {
node, _, isNode, err := delivery.ParseHaveKey(key)
if err != nil || !isNode {
continue
}
h, err := delivery.DecodeHaveNode(value)
if err != nil {
continue
}
if a, ok := h.Candidates[group]; ok {
out[node] = &a
}
}
return out
}
// online is, by node, whether the watchman holds the node online.
func (w *world) online() map[string]bool {
out := map[string]bool{}
@@ -563,16 +585,6 @@ func (w *world) value(bucket, key string) []byte {
return w.values[bucket][key]
}
// grantsBucket is the bucket the grants are read from now (delivery.GrantSource): the grants part's while the controller's
// lease is handed away, as the lease bucket's key says it (buscompose.LeaseHandedAway), the controller's in every other
// case, a lease not held here included.
func (w *world) grantsBucket() string {
raw := w.value(buscompose.LeaseBucket, buscompose.ControllerLeaseKey)
away, _ := buscompose.LeaseHandedAway(raw, raw != nil, nil)
bucket, _ := delivery.GrantSource(away)
return bucket
}
// has says whether a bucket holds a key now.
func (w *world) has(bucket, key string) bool {
w.mu.Lock()
@@ -6,9 +6,7 @@ import (
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-assigner/internal/assign"
@@ -50,7 +48,6 @@ func pull(repo string, number int, modules map[string]delivery.CandidateModule)
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
// another repository as defined twice.
func TestTheChangeOfACandidate(t *testing.T) {
theWorldCountsTheValuesAKeyTakes(t)
f := fakeManifests{}
web := f.add(t, "web", `{"module":"web","version":"2"}`)
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
@@ -116,7 +113,7 @@ func TestAManifestFallingBackToAPredefinedKeyIsReadable(t *testing.T) {
}
// Review round 1 (items 3 and 4): the world counts the values a key takes, so a have written again after an answer is seen.
func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
func TestTheWorldCountsTheValuesAKeyTakes(t *testing.T) {
w := newWorld()
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
w.put(delivery.HaveBucket, "ace.web", []byte(`{"v":1}`))
@@ -127,40 +124,4 @@ func theWorldCountsTheValuesAKeyTakes(t *testing.T) {
if got := w.changed(delivery.HaveBucket, "gone"); got != 0 {
t.Errorf("changed = %d for a key never seen", got)
}
// Issue 774: grants are read from the grants part's bucket only while the controller's lease is handed away: a grant
// only the part holds lets a node's put through then, and with a controller's lease (or none, or an unreadable one) it
// holds it, a grant only the controller holds being the one that counts instead.
fp := delivery.SourcePrefix2 + strings.Repeat("d", 64)
gkey, _ := delivery.ControllerGrantKey("web", fp)
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{Accounts: []string{"ace.web"}})
for _, row := range []struct {
name, lease string
inPart, inController bool
bucket string
granted bool
}{
{"handed away, the part's grant", `{"handed-to":"the operator's hand-over"}`, true, false, delivery.PartGrantsBucket, true},
{"handed away, only the controller's grant", `{"handed-to":"x"}`, false, true, delivery.PartGrantsBucket, false},
{"a controller holds it, only the part's grant", `{"instance":"novox/1","epoch":3}`, true, false, delivery.ControllerGrantsBucket, false},
{"a controller holds it, its own grant", `{"instance":"novox/1"}`, false, true, delivery.ControllerGrantsBucket, true},
{"no lease read", ``, true, false, delivery.ControllerGrantsBucket, false},
{"an unreadable lease", `[1]`, true, false, delivery.ControllerGrantsBucket, false},
} {
p := newPasser(buckets{}, time.Now())
if row.lease != "" {
p.w.put(buscompose.LeaseBucket, buscompose.ControllerLeaseKey, []byte(row.lease))
}
if row.inPart {
p.w.put(delivery.PartGrantsBucket, gkey, grant)
}
if row.inController {
p.w.put(delivery.ControllerGrantsBucket, gkey, grant)
}
if got := p.w.grantsBucket(); got != row.bucket {
t.Errorf("%s: grants are read from %s, want %s", row.name, got, row.bucket)
}
if _, granted := p.composedFor("ace", "web", fp); granted != row.granted {
t.Errorf("%s: granted = %v, want %v", row.name, granted, row.granted)
}
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+4 -2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648 h1:XoMsgg9kSgjQWa86NZM8h8OpxJ2kovxcMS2C0i6WSVU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027201213-7a8e9f970648/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -0,0 +1,242 @@
package assign
import (
"fmt"
"reflect"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A change's own placement (novox/hq ADR 0559 §5, issue 1126): a catalogue pull request that changes placement files has
// the indexer's placement candidates, one key `<group>.<node>` for each node whose file at its head differs from the
// trunk's. The check composes the change with those placements laid over the trunk's (frombus, Scope.Placements), counts
// every such node among the machines it touches, and then reads each placed node's own answer in its `<node>.node`
// have: it fails on what a node composes differently that the change does not intend, on a refusal, and on a node
// placed at the head or on the trunk that has not answered at the head within AnswerBound of the keys being written.
// AnswerBound is how long the check waits for every placed node's answer after the candidate keys were written (ADR 0559
// §5): a node asleep or away is named past it, never passed. A variable so a test on the bus can shorten it.
var AnswerBound = 10 * time.Minute
// ChangePlacement is the change's own placement, read from the indexer's placement candidates of its group.
type ChangePlacement struct {
// Placements is by node the candidate placement; nil when the head removes the node's file.
Placements map[string]*delivery.Placement
// Revisions is by node the revision of its candidate key: the answer to it names that revision.
Revisions map[string]uint64
// Head is the catalogue pull request's head the candidates were read at, and Written when the newest key was written.
Head string
Written time.Time
// Input is the trunk's composition input with Placements laid over it (frombus.Input, Scope.Placements), its catalogue
// the trunk's; nil with Unread saying why when it could not be read.
Input *compose.Input
Unread string
}
// Nodes is every node the change's placement names, sorted.
func (p *ChangePlacement) Nodes() []string {
if p == nil {
return nil
}
out := make([]string, 0, len(p.Placements))
for node := range p.Placements {
out = append(out, node)
}
sort.Strings(out)
return out
}
// Intended is, by node the trunk places, the modules whose composition the change's placement changes: those it adds,
// removes or whose placed entry differs, and those whose composition (with the trunk's catalogue, the same stand-ins
// either side) differs between the trunk's input and the change's (delivery.CompareComposed), the node's own named
// "node". A node the trunk does not place yet has none (ADR 0559 §5: at the seed every node must answer the same). fps
// gives the build each module is composed at, the trunk's. A node whose compositions cannot be made either side keeps
// the modules its placement entries name, and is said in notes.
func Intended(trunk *compose.Input, trunkPlaced map[string]delivery.Placement, cp *ChangePlacement, fps map[string]string) (intended map[string]map[string]bool, notes []string) {
intended = map[string]map[string]bool{}
if cp == nil {
return intended, nil
}
for _, node := range cp.Nodes() {
before, placed := trunkPlaced[node]
if !placed {
continue
}
set := map[string]bool{}
after := cp.Placements[node]
var afterModules []delivery.PlacedModule
if after != nil {
afterModules = after.Modules
}
entries := map[string]delivery.PlacedModule{}
for _, m := range before.Modules {
entries[m.Module] = m
}
seen := map[string]bool{}
for _, m := range afterModules {
seen[m.Module] = true
if e, ok := entries[m.Module]; !ok || !reflect.DeepEqual(e, m) {
set[m.Module] = true
}
}
for name := range entries {
if !seen[name] {
set[name] = true
}
}
if trunk != nil && cp.Input != nil {
was, _, _, errB := compose.CompositionsShadow(*trunk, node, fps)
now, _, _, errA := compose.CompositionsShadow(*cp.Input, node, fps)
if errA == nil && errB == nil {
running := make([]delivery.Composed, 0, len(was))
for _, c := range was {
running = append(running, c)
}
cmp := delivery.CompareComposed(now, running)
for _, d := range cmp.Differs {
set[d.Module] = true
}
for _, d := range cmp.Added {
set[d.Module] = true
}
for _, m := range cmp.Removed {
set[m] = true
}
} else {
notes = append(notes, fmt.Sprintf("what the change intends on %s is read from its placement file alone: its compositions cannot be made (trunk: %v; change: %v)", node, errB, errA))
}
}
intended[node] = set
}
return intended, notes
}
// Answered is what the placed nodes answered: the failures to name, the nodes that have not answered, and the notes.
type Answered struct {
Fails []string
Silent []string
Machine string
Waits bool
Until time.Time
Warnings []string
}
// JudgeAnswers reads each placed node's answer to the change's placement (ADR 0559 §5). placed is every node placed at
// the head or on the trunk; answers is by node its HaveNode.Candidates entry for the group, nil when it has none; online
// says whether the watchman holds the node online. An answer counts when it names the candidates' head and, for a node
// with a key of its own, that key's revision. It fails on a refusal, on a module that differs, is added or is removed
// that intended does not name, and, from Written plus AnswerBound on, on every node with no answer that counts, named,
// asleep or not; before that the check waits for them.
func JudgeAnswers(cp *ChangePlacement, placed []string, answers map[string]*delivery.CandidateAnswer, intended map[string]map[string]bool, online map[string]bool, now time.Time) Answered {
var out Answered
if cp == nil {
return out
}
failAt := func(node, words string) {
if out.Machine == "" {
out.Machine = node
}
out.Fails = append(out.Fails, words)
}
for _, node := range placed {
a := answers[node]
rev, keyed := cp.Revisions[node]
if a == nil || a.Head != cp.Head || (keyed && a.Revision != rev) {
out.Silent = append(out.Silent, node)
continue
}
if a.Refused != "" {
failAt(node, fmt.Sprintf("%s cannot compose the candidate placement: %s", node, a.Refused))
continue
}
for _, r := range a.Refusals {
failAt(node, fmt.Sprintf("%s refuses the candidate placement for %s: %s", node, r.Module, r.Why))
}
want := intended[node]
for _, d := range a.Differs {
if !want[d.Module] {
failAt(node, fmt.Sprintf("%s composes %s differently with the candidate placement, which the change does not intend: %s", node, d.Module, d.Why))
}
}
for _, d := range a.Added {
if !want[d.Module] {
failAt(node, fmt.Sprintf("%s would run %s with the candidate placement, which the change does not intend", node, d.Module))
}
}
for _, m := range a.Removed {
if !want[m] {
failAt(node, fmt.Sprintf("%s would stop running %s with the candidate placement, which the change does not intend", node, m))
}
}
}
if len(out.Silent) == 0 {
return out
}
out.Until = cp.Written.Add(AnswerBound)
named := make([]string, len(out.Silent))
for i, node := range out.Silent {
named[i] = node
if !online[node] {
named[i] = node + " (offline)"
}
}
if now.Before(out.Until) {
out.Waits = true
out.Warnings = append(out.Warnings, fmt.Sprintf("waits for the answers of %s to the candidate placement at %s, until %s",
strings.Join(named, ", "), short(cp.Head), out.Until.UTC().Format(time.RFC3339)))
return out
}
for _, node := range out.Silent {
if out.Machine == "" {
out.Machine = node
}
}
out.Fails = append(out.Fails, fmt.Sprintf("%s %s (%s after the candidate keys were written at %s)",
strings.Join(named, ", "), delivery.CompositionUnanswered, AnswerBound, cp.Written.UTC().Format(time.RFC3339)))
return out
}
// PlacedNodes is every node placed at the head or on the trunk: the trunk's placed nodes with the change's placement laid
// over them (a node whose file the head removes stays, as the trunk places it), sorted.
func PlacedNodes(trunkPlaced map[string]delivery.Placement, cp *ChangePlacement) []string {
set := map[string]bool{}
for node := range trunkPlaced {
set[node] = true
}
if cp != nil {
for node, p := range cp.Placements {
if p != nil {
set[node] = true
}
}
}
out := make([]string, 0, len(set))
for node := range set {
out = append(out, node)
}
sort.Strings(out)
return out
}
// WithAnswers is v with the placed nodes' answers judged in: a fail adds its failures (the words of nodes not heard end
// with delivery.CompositionUnanswered, so the checker takes a later composition of the same heads in when they answer), a
// wait makes a passing verdict wait (a composition that waits composed no machine), and a failing one stands.
func WithAnswers(v delivery.CompositionVerdict, a Answered) delivery.CompositionVerdict {
if len(a.Fails) > 0 {
if v.State == delivery.CompositionFail {
v.Words += "; " + strings.Join(a.Fails, "; ")
} else {
v.State, v.Words, v.Machine = delivery.CompositionFail, strings.Join(a.Fails, "; "), a.Machine
}
return v
}
if a.Waits && v.State == delivery.CompositionPass {
return delivery.CompositionVerdict{Group: v.Group, Heads: v.Heads, State: delivery.CompositionWaits, SDK: v.SDK, Took: v.Took,
Words: strings.Join(a.Warnings, "; ")}
}
return v
}
@@ -59,13 +59,15 @@ type Change struct {
// Members the head of each member by repository: what a composer at the group's mesh-sdk reads (issues 1622, 1623).
Digests map[string]string
Members map[string]string
// Placement is the change's own placement, read from the indexer's placement candidates of the group (ADR 0559 §5);
// nil when the group changes no placement file.
Placement *ChangePlacement
}
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
// touches no machine. A change of placement is no change of a repository (ADR 0567 §4): the seat
// mesh-placement judges it.
// touches no machine. Every node whose placement the change changes is touched too (ADR 0559 §5, changing ADR 0446 §3).
func Touched(m Mesh, c Change) []string {
moved := map[string]bool{}
for name := range c.Manifests {
@@ -99,6 +101,11 @@ func Touched(m Mesh, c Change) []string {
}
}
}
for _, node := range c.Placement.Nodes() {
if !slices.Contains(out, node) {
out = append(out, node)
}
}
sort.Strings(out)
return out
}
@@ -171,6 +178,25 @@ func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.Composit
delete(changed, name)
}
after := input(changed, m.Assigned)
if c.Placement != nil {
// A change of placement is composed over the input the nodes compose from, its placement laid over the trunk's
// (frombus, ADR 0559 §5): the trunk as the nodes compose it where the catalogue places them, the change's
// manifests over the trunk's catalogue.
if m.Input != nil {
trunk = *m.Input
trunk.Catalogue = withOverlay(m.Input.Catalogue, m.Manifests)
}
if c.Placement.Input == nil {
failAt("", "the change's placement cannot be composed: the input with it cannot be read from the bus: "+c.Placement.Unread)
} else {
after = *c.Placement.Input
cat := withOverlay(c.Placement.Input.Catalogue, changed)
for _, name := range c.Removed {
delete(cat, name)
}
after.Catalogue = cat
}
}
for _, node := range machines {
_, before := compose.Resolve(trunk, node)
_, now := compose.Resolve(after, node)
@@ -4,6 +4,7 @@ import (
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -57,6 +58,79 @@ func TestAMachineTheChangeBreaksFails(t *testing.T) {
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
}
// ADR 0559 §5 (issue 1126): a change of placement composes its placement laid over the trunk's input and fails naming
// the node it breaks: a module no manifest names placed on g14; ace,
// is no longer composed with it; a placement whose input cannot be read fails; a placement that breaks nothing passes, g14 touched.
m := meshOf()
placed := func(assigned map[string][]string) *ChangePlacement {
in := input(m.Manifests, assigned)
return &ChangePlacement{Placements: map[string]*delivery.Placement{"g14": {Node: "g14"}}, Input: &in}
}
for _, row := range []struct {
name string
cp *ChangePlacement
state string
machine string
words string
}{
{"a placement that breaks g14", placed(map[string][]string{"ace": {"mail", "store"}, "g14": {"web", "ghost"}}), delivery.CompositionFail, "g14", "ghost"},
{"a placement whose input is not read", &ChangePlacement{Placements: map[string]*delivery.Placement{"g14": nil}, Unread: "no record of g14"}, delivery.CompositionFail, "", "no record of g14"},
{"a placement that breaks nothing", placed(map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}}), delivery.CompositionPass, "", ""},
} {
v := Compose(m, Change{Group: "g", Heads: "0123456789ab", Placement: row.cp}, "sdk", 0)
if v.State != row.state || v.Machine != row.machine || !strings.Contains(v.Words, row.words) || !slices.Contains(v.Machines, "g14") {
t.Errorf("%s: %+v", row.name, v)
}
}
// ADR 0559 §5: the placed nodes' answers. ace is placed on the trunk and its candidate moves web there (intended);
// g14 is not placed on the trunk (the seed), so nothing is intended there; novox is placed and unchanged.
at := time.Date(2026, 10, 27, 2, 0, 0, 0, time.UTC)
cp := &ChangePlacement{Head: "h2", Written: at, Revisions: map[string]uint64{"ace": 7, "g14": 8},
Placements: map[string]*delivery.Placement{"ace": {Node: "ace"}, "g14": {Node: "g14"}}}
intended := map[string]map[string]bool{"ace": {"web": true}}
same := func(rev uint64) *delivery.CandidateAnswer {
return &delivery.CandidateAnswer{Head: "h2", Revision: rev, Verdict: delivery.CandidateSame, Same: []string{"mail"}}
}
placedNodes := []string{"ace", "g14", "novox"}
for _, row := range []struct {
name string
answers map[string]*delivery.CandidateAnswer
now time.Time
state string
words string
}{
{"all the same", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": same(8), "novox": same(0)}, at, delivery.CompositionPass, ""},
{"a module added where intended", map[string]*delivery.CandidateAnswer{"ace": {Head: "h2", Revision: 7, Verdict: delivery.CandidateDiffers,
Added: []delivery.ModuleAdded{{Module: "web", Unissued: []string{"its own secret"}}}}, "g14": same(8), "novox": same(0)}, at, delivery.CompositionPass, ""},
{"a module differing where not intended", map[string]*delivery.CandidateAnswer{"ace": {Head: "h2", Revision: 7, Verdict: delivery.CandidateDiffers,
Differs: []delivery.ModuleDiffers{{Module: "mail", Why: "the port differs"}}}, "g14": same(8), "novox": same(0)}, at, delivery.CompositionFail, "ace composes mail differently"},
{"a node's first placement file with a module differing", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": {Head: "h2", Revision: 8, Verdict: delivery.CandidateDiffers,
Differs: []delivery.ModuleDiffers{{Module: "web", Why: "a binding differs"}}}, "novox": same(0)}, at, delivery.CompositionFail, "g14 composes web differently"},
{"a refusal", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": {Head: "h2", Revision: 8, Verdict: delivery.CandidateRefused,
Refusals: []delivery.ModuleRefused{{Module: "web", Why: "no own secret"}}}, "novox": same(0)}, at, delivery.CompositionFail, "g14 refuses"},
{"a node not answering, within its bound", map[string]*delivery.CandidateAnswer{"ace": same(7), "novox": same(0)}, at.Add(AnswerBound - time.Second), delivery.CompositionWaits, "g14 (offline)"},
{"an answer of an earlier revision, past its bound", map[string]*delivery.CandidateAnswer{"ace": same(6), "g14": same(8), "novox": same(0)}, at.Add(AnswerBound), delivery.CompositionFail, "ace " + delivery.CompositionUnanswered},
{"its later answer", map[string]*delivery.CandidateAnswer{"ace": same(7), "g14": same(8), "novox": same(0)}, at.Add(time.Hour), delivery.CompositionPass, ""},
} {
pass := delivery.CompositionVerdict{Group: "g", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "sdk", Machines: []string{"ace", "g14"}}
v := WithAnswers(pass, JudgeAnswers(cp, placedNodes, row.answers, intended, map[string]bool{"ace": true, "novox": true}, row.now))
if v.State != row.state || !strings.Contains(v.Words, row.words) {
t.Errorf("%s: %+v", row.name, v)
}
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
t.Errorf("%s: the verdict cannot be written: %v", row.name, err)
}
}
trunk := map[string]delivery.Placement{"ace": {Node: "ace", Modules: []delivery.PlacedModule{{Module: "mail"}}}, "novox": {Node: "novox"}}
if got := PlacedNodes(trunk, &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": nil, "g14": {Node: "g14"}}}); !slices.Equal(got, []string{"ace", "g14", "novox"}) {
t.Errorf("placed at the head or on the trunk = %v", got)
}
got, _ := Intended(nil, trunk, &ChangePlacement{Placements: map[string]*delivery.Placement{"ace": {Node: "ace", Modules: []delivery.PlacedModule{{Module: "mail", Ports: map[string]int{"25": 2525}}, {Module: "web"}}}, "g14": {Node: "g14", Modules: []delivery.PlacedModule{{Module: "web"}}}}}, nil)
if !got["ace"]["mail"] || !got["ace"]["web"] || len(got["g14"]) != 0 {
t.Errorf("intended = %v, want ace's mail and web, nothing on g14 the trunk does not place", got)
}
}
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
@@ -109,6 +183,10 @@ func TestTouched(t *testing.T) {
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
t.Errorf("touched by store with a consumer on g14 = %v", got)
}
// ADR 0559 §5: every node whose placement the change changes is touched, its file removed or not.
if got := Touched(m, Change{Placement: &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": nil, "shanks": {Node: "shanks"}}}}); !slices.Equal(got, []string{"novox", "shanks"}) {
t.Errorf("touched by a placement = %v", got)
}
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
t.Errorf("an added module assigned nowhere: %+v", v)
@@ -50,6 +50,17 @@ func HeadJob(m Mesh, c Change) (delivery.ComposeJob, error) {
fromClaims = len(in.Holdings) == 0
}
changeCase := composer.Case{Name: CaseChange, Manifests: change}
if c.Placement != nil {
// The change's own placement, laid over the input's in the change case (ADR 0559 §5; composer.Case's Placements):
// a node the input does not hold yet is added bare, so the composer places it.
changeCase.Placements = c.Placement.Placements
in.Nodes = slices.Clone(in.Nodes)
for _, node := range c.Placement.Nodes() {
if !slices.ContainsFunc(in.Nodes, func(n compose.Node) bool { return n.Name == node }) {
in.Nodes = append(in.Nodes, compose.Node{Name: node})
}
}
}
req := composer.Request{V: composer.Version, Input: in, Depth: composer.DepthResolve,
HoldingsFromClaims: fromClaims, Machines: Touched(m, c),
Cases: []composer.Case{{Name: CaseTrunk, Manifests: trunk}, changeCase}}
@@ -120,6 +120,22 @@ func TestAGroupWithAnSDKMemberAsksTheComposerAtItsHead(t *testing.T) {
len(placed.Input.Holdings) != 1 || placed.HoldingsFromClaims {
t.Errorf("the placed check's request = %+v, holdings from claims %v", placed.Input, placed.HoldingsFromClaims)
}
// ADR 0559 §5: a change of placement composes its own placement in the change case alone (composer.Case's Placements),
// a node the input does not hold yet added bare so the composer places it, and the node touched.
c.Placement = &ChangePlacement{Placements: map[string]*delivery.Placement{"novox": {Node: "novox"}}}
candJob, err := HeadJob(m, c)
if err != nil {
t.Fatal(err)
}
cand, err := composer.Decode(candJob.Request)
if err != nil {
t.Fatal(err)
}
if cand.Cases[0].Placements != nil || cand.Cases[1].Placements["novox"] == nil || len(cand.Input.Nodes) != 2 || cand.Input.Nodes[1].Name != "novox" ||
!slices.Equal(cand.Machines, []string{"g14", "novox"}) || len(m.Input.Nodes) != 1 {
t.Errorf("the placement's request = cases %+v, nodes %+v, machines %v", cand.Cases, cand.Input.Nodes, cand.Machines)
}
}
// 1622/AC2, AC3: a catalogue member using a field only the SDK head reads passes with the head's composer, and the
@@ -2,49 +2,33 @@ package assign
import (
"fmt"
"slices"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Until ADR 0547's third milestone two sources name which module runs on which node: the controller's assignments and
// the seat mesh-placement's placement (ADR 0549 §5, ADR 0567 §8). The assigner writes wants only for a pair both name:
// the indexer's placement, read from the catalogue (ADR 0549 §5). The assigner writes wants only for a pair both name:
// for a pair one source names alone it writes no want and touches none it wrote before, so nothing moves and nothing is
// removed on one source's word, and it raises a condition naming the pair until both agree. After an approved proposal
// the assigner keeps the controller in step itself, calling its assign, unassign, pin and unpin; a call refused or not
// yet answered leaves the pair differing, and the condition names the call (Pair.Call). Before the first placement key
// is written the controller's assignments are the one source, as before. From the lease write (the controller's lease
// handed away) the placement is the one source.
// removed on one source's word, and it raises a condition naming the pair until both agree. Before the first placement
// file reaches the indexer (no placement key at all) the controller's assignments are the one source, as before.
// Pair is a module on a node that one source names and the other does not; Only says which: OnlyAssignments or
// OnlyPlacement.
type Pair struct {
Node, Module, Only string
// Call is what became of the assigner's call of the controller's verb for the pair, in words, when it made one.
Call string
}
// The source that names a pair alone.
const (
OnlyAssignments = "the controller's assignments"
OnlyPlacement = "the seat mesh-placement's placement"
OnlyPlacement = "the catalogue's placement"
)
// Placed is, by node, the modules both sources name, in name order, and every pair one names alone, by node then
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs. Once the controller's
// lease is handed away (handedAway, ADR 0549 §5, §6) it is the placement alone, in name order, and no pair differs: the
// assignments are then a source of nothing.
func Placed(assigned, placed map[string][]string, seeded, handedAway bool) (map[string][]string, []Pair) {
if handedAway && seeded {
out := make(map[string][]string, len(placed))
for node, modules := range placed {
out[node] = slices.Sorted(slices.Values(modules))
}
return out, nil
}
// module. With no placement read yet (seeded false) it is the assignments, and no pair differs.
func Placed(assigned, placed map[string][]string, seeded bool) (map[string][]string, []Pair) {
if !seeded {
return assigned, nil
}
@@ -95,7 +79,7 @@ func PlacementCondition(p Pair, raised bool) asks.Condition {
subject := asks.Scope{Scope: "module", ID: p.Module, Machine: p.Node}
if !raised {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared,
Resolved: fmt.Sprintf("the controller's assignments and the seat's placement agree on %s on %s.", p.Module, p.Node)}
Resolved: fmt.Sprintf("the controller's assignments and the catalogue's placement agree on %s on %s.", p.Module, p.Node)}
}
headline := fmt.Sprintf("%s on %s is placed by one source only", p.Module, p.Node)
if n := len([]rune(headline)); n > asks.HeadlineLength {
@@ -103,12 +87,9 @@ func PlacementCondition(p Pair, raised bool) asks.Condition {
}
var needs string
if p.Only == OnlyAssignments {
needs = fmt.Sprintf("propose placing %s on %s (mesh-placement.propose), or unassign it at the controller", p.Module, p.Node)
needs = fmt.Sprintf("add %s to the catalogue's placement file of %s, or unassign it at the controller", p.Module, p.Node)
} else {
needs = fmt.Sprintf("assign %s to %s at the controller, as the approved placement says, or propose taking it off", p.Module, p.Node)
}
if p.Call != "" {
needs += "; the assigner's call: " + p.Call
needs = fmt.Sprintf("assign %s to %s at the controller, as the merged placement file says", p.Module, p.Node)
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
@@ -116,17 +97,3 @@ func PlacementCondition(p Pair, raised bool) asks.Condition {
"name a module on a node (ADR 0549 §5): the assigner writes no want for it there and leaves the one it wrote before, so "+
"nothing moves or is removed. This clears once both name it or neither does.", asks.NeedsYou, needs, p.Only, p.Module, p.Node)}
}
// SeatWaitsCondition is the assigner's word on the seat mesh-placement while a state of the seat is not bound (ADR 0567
// §1): raised naming each, cleared when none is left.
func SeatWaitsCondition(unbound []string) asks.Condition {
key := delivery.AssignerModule + ".seat-states"
subject := asks.Scope{Scope: "module", ID: delivery.AssignerModule}
if len(unbound) == 0 {
return asks.Condition{Key: key, Subject: subject, State: asks.StateCleared, Resolved: "the seat mesh-placement holds all its states."}
}
why := "the seat waits for its states: " + strings.Join(unbound, ", ") + " are not on the bus or not granted yet"
return asks.Condition{Key: key, Severity: asks.SeverityWarning, Subject: subject, State: asks.StateRaised,
Summary: "the seat waits for its states", Headline: "the seat waits for its states",
Explanation: asks.NothingToDo + " " + why + "; each is bound at the next pass once it is, with no restart."}
}
@@ -1,296 +0,0 @@
package assign
import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The seat mesh-placement (novox/hq ADR 0567), held by the assigner beside its wants: placement is its state
// `placement`, changed only by a proposal (§2) the nodes it touches answer (§3), judged (§4) and written on the
// operator's Approve at the revision it was laid over (§5). What here decides; the tools and the passes do.
// ProposeIn is what a proposal is laid over: its id, who and why, its changes; every node's current placement and the
// revision of its key; which modules the indexer holds and which nodes have a record; and, for the seed, the export's
// digest.
type ProposeIn struct {
ID, By, Why string
At time.Time
Changes []delivery.PlacementProposalChange
Current map[string]delivery.Placement
Revisions map[string]uint64
Held func(module string) bool
Recorded func(node string) bool
Seed bool
ExportDigest string
// Leased says the lease write of ADR 0547 has happened (the controller's lease handed away, mesh-sdk
// buscompose.LeaseHandedAway; issue 774), after which the controller's assignments are read no more (§8): until then
// the holder keeps the controller in step and refuses what its verbs cannot take.
Leased bool
}
// Propose lays a proposal over the current keys (§2): the proposal (state answering, each changed node's base revision),
// the laid placement of every node whose key it changes, and every finding, which refuses it whole: a change the
// controller's verbs cannot take before the lease write (§8), a change that does not fit its node, and what
// CheckPlacements refuses of the result laid over every placement (two modules on one host port, a provider not placed,
// a seat held once held twice, two nodes with one address, two hubs, a module the indexer does not hold, a node with no
// record). A key whose value the proposal leaves as it is changes nothing and is not laid.
func Propose(in ProposeIn) (delivery.PlacementProposal, map[string]*delivery.Placement, []string) {
p := delivery.PlacementProposal{ID: in.ID, By: strings.TrimSpace(in.By), Why: strings.TrimSpace(in.Why), At: in.At.UTC(),
Changes: in.Changes, Seed: in.Seed, ExportDigest: in.ExportDigest, Base: map[string]uint64{}, State: delivery.ProposalAnswering}
var findings []string
if p.By == "" {
findings = append(findings, "the runtime names no caller: a proposal is made by someone")
}
if p.Why == "" {
findings = append(findings, "a proposal says why: it is shown to the operator")
}
if len(in.Changes) == 0 {
findings = append(findings, "a proposal changes something: it names no change")
}
for i, c := range in.Changes {
if (c.Op == delivery.OpSeed) != in.Seed {
findings = append(findings, fmt.Sprintf("change %d: the seed is adopt's alone, and never mixed with other changes", i+1))
continue
}
if !in.Leased {
if why := c.BeforeLease(); why != "" {
findings = append(findings, fmt.Sprintf("change %d: %s", i+1, why))
}
}
}
laid, lay := delivery.LayChanges(in.Current, in.Changes)
findings = append(findings, lay...)
all := map[string]delivery.Placement{}
for node, pl := range in.Current {
all[node] = pl
}
out := map[string]*delivery.Placement{}
for node, pl := range laid {
if before, ok := in.Current[node]; ok && delivery.SamePlacement(before, pl) {
continue
}
pl := pl
all[node] = pl
out[node] = &pl
p.Base[node] = in.Revisions[node]
}
if len(findings) == 0 && len(out) == 0 {
findings = append(findings, "the changes leave every placement as it is")
}
held, recorded := in.Held, in.Recorded
if held == nil {
held = func(string) bool { return false }
}
if recorded == nil {
recorded = func(string) bool { return false }
}
for _, r := range delivery.CheckPlacements(all, held, recorded) {
findings = append(findings, r)
}
return p, out, findings
}
// Write is one key a warrant writes (§5): the node, its key, the revision it is written at (0: it has none, so it is
// created) and its value (nil: it is deleted).
type Write struct {
Node, Key string
Base uint64
Value *delivery.Placement
}
// Writes is every key a proposal writes, by node order.
func Writes(p delivery.PlacementProposal, laid map[string]*delivery.Placement) ([]Write, error) {
var out []Write
for _, node := range p.Nodes() {
key, err := delivery.PlacementKey(node)
if err != nil {
return nil, err
}
out = append(out, Write{Node: node, Key: key, Base: p.Base[node], Value: laid[node]})
}
return out, nil
}
// ControllerCall is one call of the controller's seat verbs the holder makes after writing an approved proposal, until
// the lease write (§8): assign, unassign, pin or unpin, with its arguments.
type ControllerCall struct {
Verb string
Args map[string]any
// Node and Module name the pair the call keeps in step (a pin names no module).
Node, Module string
}
// Key names the call, as the condition of its pair says it.
func (c ControllerCall) Key() string {
return fmt.Sprintf("mesh-controller.%s %s %s", c.Verb, c.Node, c.Module)
}
// ControllerCalls is every call that brings the controller's assignments and pins to the placement after: assign for a
// module placed on a node before not, unassign for one taken off, pin for a pin set or changed, unpin for one removed,
// each naming the proposal and its warrant in its why. In node order, unassigns before assigns, so a module moved
// between nodes is first taken off.
func ControllerCalls(before map[string]delivery.Placement, after map[string]*delivery.Placement, proposal, warrant string) []ControllerCall {
why := fmt.Sprintf("placement proposal %s, approved (%s)", proposal, warrant)
nodes := make([]string, 0, len(after))
for node := range after {
nodes = append(nodes, node)
}
sort.Strings(nodes)
var off, on, pins []ControllerCall
for _, node := range nodes {
was, now := before[node], delivery.Placement{}
if a := after[node]; a != nil {
now = *a
}
had, has := map[string]bool{}, map[string]bool{}
for _, m := range was.Modules {
had[m.Module] = true
}
for _, m := range now.Modules {
has[m.Module] = true
if !had[m.Module] {
on = append(on, ControllerCall{Verb: "assign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
}
}
for _, m := range was.Modules {
if !has[m.Module] {
off = append(off, ControllerCall{Verb: "unassign", Node: node, Module: m.Module, Args: map[string]any{"node": node, "module": m.Module}})
}
}
for _, provision := range sortedKeys(now.Pins) {
to := now.Pins[provision]
if old, ok := was.Pins[provision]; !ok || old != to {
pins = append(pins, ControllerCall{Verb: "pin", Node: node, Module: provision,
Args: map[string]any{"node": node, "provision": provision, "from": to.Node, "module": to.Module}})
}
}
for _, provision := range sortedKeys(was.Pins) {
if _, ok := now.Pins[provision]; !ok {
pins = append(pins, ControllerCall{Verb: "unpin", Node: node, Module: provision, Args: map[string]any{"node": node, "provision": provision}})
}
}
}
out := append(append(off, on...), pins...)
for i := range out {
out[i].Args["why"] = why
}
return out
}
// PlacementAct is the act an approved proposal's yes binds: writing exactly its changes at its base revisions.
func PlacementAct(p delivery.PlacementProposal) asks.Act {
return asks.Act{"placement": "write", "proposal": p.ID, "changes": delivery.ChangesDigest(p)}
}
// PlacementAsking is a proposal's asking number: each judging that raises its ask again (again, a key moved) is
// another asking, under an id of its own.
func PlacementAsking(p delivery.PlacementProposal, at time.Time) int {
return int(at.Unix())
}
// PlacementAsk is the authorising ask about a proposal (§5): an Approve that binds its act, so only a channel that
// proves who answered takes it, with the brief of the verdict by node (judge.Brief) as its explanation.
func PlacementAsk(key string, p delivery.PlacementProposal, brief string, asking int, now time.Time) (delivery.AssignerAsk, error) {
binds, err := asks.ActDigest(PlacementAct(p))
if err != nil {
return delivery.AssignerAsk{}, err
}
nodes := p.Nodes()
headline := fmt.Sprintf("Change placement on %s?", strings.Join(nodes, ", "))
if p.Seed {
headline = "Take placement over from the controller?"
}
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = fmt.Sprintf("Change placement on %d machines?", len(nodes))
}
return delivery.AssignerAsk{Ask: asks.Ask{
ID: delivery.AskingID(key, asking),
Headline: headline,
Explanation: brief,
Options: []asks.Option{
{ID: OptionYes, Label: "Approve", Does: "the placement of " + strings.Join(nodes, ", ") + " is written as proposed",
Level: asks.Approve, Binds: binds},
{ID: OptionNo, Label: "Decline", Does: "nothing is written, and the proposal ends", Level: asks.Acknowledge},
},
Who: asks.Operator,
Expires: now.Add(asks.ApproveLasts),
OnExpiry: "nothing is written, and the proposal ends",
About: key,
}, Asking: asking}, nil
}
// ExportOf reads the controller's answer to placement-export (§7): its verb answers what the command printed, the
// JSON of every node's placement, in `output` (the controller's verbAnswer; `answer` when it carries it as data). A
// refused or unreadable answer is an error naming it.
func ExportOf(raw []byte) (map[string]delivery.Placement, error) {
var a struct {
Output string `json:"output"`
OK *bool `json:"ok"`
Answer json.RawMessage `json:"answer"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the controller's placement-export answered no verb answer: %w", err)
}
if a.OK != nil && !*a.OK {
return nil, fmt.Errorf("the controller's placement-export refused: %s", strings.TrimSpace(a.Output))
}
var byNode map[string]json.RawMessage
if len(a.Answer) > 0 && string(a.Answer) != "null" {
if err := json.Unmarshal(a.Answer, &byNode); err != nil {
return nil, fmt.Errorf("the controller's placement-export is no placement by node: %w", err)
}
} else if err := json.NewDecoder(strings.NewReader(a.Output)).Decode(&byNode); err != nil {
// The command prints its JSON first; what its standard error said after it is left.
return nil, fmt.Errorf("the controller's placement-export printed no placement by node: %w", err)
}
out := map[string]delivery.Placement{}
for node, rawP := range byNode {
p, err := delivery.DecodePlacement(rawP)
if err != nil {
return nil, fmt.Errorf("the controller's placement-export of %s cannot be read: %w", node, err)
}
if p.Node != node {
return nil, fmt.Errorf("the controller's placement-export names %s under %s", p.Node, node)
}
out[node] = p
}
if len(out) == 0 {
return nil, fmt.Errorf("the controller's placement-export names no node")
}
return out, nil
}
// SeedChanges is the seed's changes: every node's placement of the export, whole, in node order.
func SeedChanges(export map[string]delivery.Placement) []delivery.PlacementProposalChange {
var out []delivery.PlacementProposalChange
for _, node := range sortedKeys(export) {
p := export[node]
out = append(out, delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: node, Whole: &p})
}
return out
}
// Rest is what a proposal leaves to write once the keys of written are: its changes of the other nodes, to be laid again
// over the keys as the write left them (§5: a key moved while it was written). False when nothing is left.
func Rest(p delivery.PlacementProposal, written map[string]*delivery.Placement) (delivery.PlacementProposal, bool) {
rest := p
rest.Changes, rest.Base = nil, map[string]uint64{}
for _, c := range p.Changes {
if _, done := written[c.Node]; !done {
rest.Changes = append(rest.Changes, c)
}
}
for node, base := range p.Base {
if _, done := written[node]; !done {
rest.Base[node] = base
}
}
rest.Must, rest.Verdict, rest.Ask, rest.State = nil, nil, "", delivery.ProposalAnswering
return rest, len(rest.Changes) > 0
}
@@ -1,157 +0,0 @@
package assign
import (
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0567 §2, §5, §7, §8: what the seat's holder decides of a proposal: every refusal named at once, a valid one laid
// with the base revision of each key it changes, the writes at those revisions, the controller's calls that keep its
// assignments in step, the authorising ask bound to the changes, and the export read as the controller answers it.
func TestThePlacementSeatDecides(t *testing.T) {
anchor := delivery.Placement{V: delivery.Version, Node: "anchor", Decided: delivery.NodeDecided{Hub: true, Address: "10.0.0.1"},
Modules: []delivery.PlacedModule{{Module: "store", Ports: map[string]int{"5432": 5432}}, {Module: "web"}}}
laptop := delivery.Placement{V: delivery.Version, Node: "laptop", Decided: delivery.NodeDecided{Address: "10.0.0.2"}, Modules: []delivery.PlacedModule{{Module: "shell"}}}
in := func(changes ...delivery.PlacementProposalChange) ProposeIn {
return ProposeIn{ID: "p-one", By: "operator@laptop", Why: "move web", At: time.Unix(1, 0), Changes: changes,
Current: map[string]delivery.Placement{"anchor": anchor, "laptop": laptop}, Revisions: map[string]uint64{"anchor": 4, "laptop": 9},
Held: func(m string) bool { return m != "ghost" },
Recorded: func(n string) bool { return n == "anchor" || n == "laptop" }}
}
move := []delivery.PlacementProposalChange{{Op: delivery.OpTakeOff, Node: "anchor", Module: "web"}, {Op: delivery.OpPlace, Node: "laptop", Module: "web"}}
p, laid, findings := Propose(in(move...))
if len(findings) != 0 || p.State != delivery.ProposalAnswering || p.Base["anchor"] != 4 || p.Base["laptop"] != 9 ||
!slices.Equal(laid["laptop"].Assigned(), []string{"shell", "web"}) || !slices.Equal(laid["anchor"].Assigned(), []string{"store"}) {
t.Fatalf("a move: %+v, %+v, %v", p, laid, findings)
}
if _, err := delivery.EncodePlacementProposal(p); err != nil {
t.Fatalf("the proposal cannot be written: %v", err)
}
for name, c := range map[string]struct {
in ProposeIn
want string
}{
"a port before the lease": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}}), "lease write"},
"a holding": {in(delivery.PlacementProposalChange{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "mesh-store", Scope: "mesh"}), "lease write"},
"a binding": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "web", Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}}), "lease write"},
"a config place": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}}), "lease write"},
"a config access": {in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", MeshWide: map[string]any{"accesses": []any{}}}), "lease write"},
"a module not held": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "laptop", Module: "ghost"}), "the indexer holds no module ghost"},
"a node with no record": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "stranger", Module: "web"}), "stranger is a node with no record"},
"nothing changes": {in(delivery.PlacementProposalChange{Op: delivery.OpPlace, Node: "anchor", Module: "web"}), "leave every placement as it is"},
"no why": {func() ProposeIn { i := in(move...); i.Why = " "; return i }(), "says why"},
"no caller": {func() ProposeIn { i := in(move...); i.By = ""; return i }(), "names no caller"},
"a seed by hand": {in(delivery.PlacementProposalChange{Op: delivery.OpSeed, Node: "laptop", Whole: &laptop}), "adopt's alone"},
"a pin of nobody placed": {in(delivery.PlacementProposalChange{Op: delivery.OpPin, Node: "laptop", Provision: "db", Provider: &delivery.Provider{Node: "laptop", Module: "store"}}), "not placed there"},
} {
if _, _, f := Propose(c.in); !slices.ContainsFunc(f, func(s string) bool { return strings.Contains(s, c.want) }) {
t.Errorf("%s: findings %v, want one naming %q", name, f, c.want)
}
}
// A module's config values may be proposed before the lease write (§13); once the lease is written every change is
// the state's alone, and none waits.
if _, _, f := Propose(in(delivery.PlacementProposalChange{Op: delivery.OpConfig, Node: "anchor", Module: "web", Values: map[string]any{"puid": 1}})); len(f) != 0 {
t.Errorf("config values before the lease write: %v", f)
}
for _, c := range []delivery.PlacementProposalChange{
{Op: delivery.OpPlace, Node: "laptop", Module: "web", Ports: map[string]int{"80": 8080}, Bindings: map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}},
{Op: delivery.OpConfig, Node: "anchor", Module: "web", Layer: map[string]any{"places": []any{}}},
{Op: delivery.OpDecide, Node: "laptop", Field: "site", Value: json.RawMessage(`"attic"`)},
{Op: delivery.OpHold, Node: "anchor", Module: "store", Claim: "node-shell", Scope: "node"},
} {
leased := in(c)
leased.Leased = true
if _, _, f := Propose(leased); len(f) != 0 {
t.Errorf("after the lease write %s is refused: %v", c.Op, f)
}
}
// A key moved while the warrant wrote: what is left to write is the other nodes' changes, to be laid and asked again.
if rest, ok := Rest(p, map[string]*delivery.Placement{"anchor": laid["anchor"]}); !ok || len(rest.Changes) != 1 || rest.Changes[0].Node != "laptop" ||
len(rest.Base) != 1 || rest.State != delivery.ProposalAnswering || rest.Must != nil {
t.Errorf("the rest = %+v, %v", rest, ok)
}
if _, ok := Rest(p, laid); ok {
t.Error("a proposal written whole leaves something to write")
}
// The seat says it waits while one of its states is not bound, naming it, and clears once all are.
for _, unbound := range [][]string{{delivery.PlacementPulsesBucket}, nil} {
c := SeatWaitsCondition(unbound)
if _, err := asks.EncodeCondition(c); err != nil || (len(unbound) > 0) != (c.State == asks.StateRaised) ||
(len(unbound) > 0 && !strings.Contains(c.Explanation, "the seat waits for its states: "+delivery.PlacementPulsesBucket)) {
t.Errorf("waiting for %v: %+v, %v", unbound, c, err)
}
}
// Issue 1126: the seed places the private network, which no repository holds; it is the mesh's own, and passes.
wired := laptop
wired.Modules = append(slices.Clone(laptop.Modules), delivery.PlacedModule{Module: "mesh-wireguard"})
seed := in(SeedChanges(map[string]delivery.Placement{"laptop": wired})...)
seed.Seed, seed.ExportDigest, seed.Current, seed.Revisions = true, "sha256:x", map[string]delivery.Placement{}, map[string]uint64{}
seed.Held = func(m string) bool { return m != "mesh-wireguard" } // the indexer holds every module but it
if _, _, f := Propose(seed); len(f) != 0 {
t.Errorf("a seed placing the private network is refused: %v", f)
}
writes, err := Writes(p, laid)
if err != nil || len(writes) != 2 || writes[0].Key != "placement.anchor" || writes[0].Base != 4 || writes[1].Base != 9 {
t.Fatalf("writes = %+v, %v", writes, err)
}
pinned := *laid["laptop"]
pinned.Pins = map[string]delivery.Provider{"db": {Node: "anchor", Module: "store"}}
calls := ControllerCalls(map[string]delivery.Placement{"anchor": anchor, "laptop": laptop},
map[string]*delivery.Placement{"anchor": laid["anchor"], "laptop": &pinned}, "p-one", "w1@t")
var said []string
for _, c := range calls {
said = append(said, c.Key())
if !strings.Contains(c.Args["why"].(string), "p-one") {
t.Errorf("%s does not name the proposal: %v", c.Key(), c.Args)
}
}
if !slices.Equal(said, []string{"mesh-controller.unassign anchor web", "mesh-controller.assign laptop web", "mesh-controller.pin laptop db"}) {
t.Errorf("the controller's calls = %v", said)
}
a, err := PlacementAsk("placement.p-one", p, "move web (proposed by operator@laptop)", 7, time.Unix(100, 0))
if err != nil || a.Ask.Check(time.Unix(100, 0)) != nil || a.Ask.Options[0].Level != asks.Approve || a.Ask.Options[0].Performs(PlacementAct(p)) != nil {
t.Fatalf("the ask = %+v, %v", a, err)
}
q := p
q.Changes = append(slices.Clone(p.Changes), delivery.PlacementProposalChange{Op: delivery.OpUnpin, Node: "laptop", Provision: "x"})
if a.Ask.Options[0].Performs(PlacementAct(q)) == nil {
t.Error("the ask's yes binds changes it was not asked about")
}
export, _ := json.Marshal(map[string]any{"anchor": anchor, "laptop": laptop})
for name, c := range map[string]struct {
raw string
want int
}{
"in output": {`{"ok":true,"output":` + quote(string(export)+"\n") + `}`, 2},
"as answer": {`{"ok":true,"output":"","answer":` + string(export) + `}`, 2},
"refused": {`{"ok":false,"output":"no store"}`, 0},
"no node": {`{"ok":true,"output":"{}"}`, 0},
"no version": {`{"ok":true,"output":"{\"anchor\":{\"node\":\"anchor\",\"modules\":[]}}"}`, 0},
} {
got, err := ExportOf([]byte(c.raw))
if len(got) != c.want || (c.want == 0) != (err != nil) {
t.Errorf("%s: %d nodes, %v", name, len(got), err)
}
}
if seed := SeedChanges(map[string]delivery.Placement{"laptop": laptop, "anchor": anchor}); len(seed) != 2 || seed[0].Node != "anchor" || seed[0].Op != delivery.OpSeed {
t.Errorf("the seed's changes = %+v", seed)
}
}
func quote(s string) string {
raw, _ := json.Marshal(s)
return string(raw)
}
@@ -89,15 +89,14 @@ func Canary(nodes []string, online map[string]bool) string {
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
// limit (ADR 0528 clause 3). Rule 5, every provider healthy, holds no want (issue 1960): a module whose provider is down
// is wanted all the same, and said as not reached (Unreached), since a site runs while the proxy in front of it is
// down and one module never blocks another.
// limit (ADR 0528 clause 3).
func Holds(m Module, node string, n Nodes) (bool, int, string) {
return holds(m, node, n, true)
}
// holds is Holds; with ordering false it leaves out the canary (3), which orders a move among nodes, and keeps those
// without which there is nothing to want: the build done, not held, the grant and the composition.
// holds is Holds; with ordering false it leaves out the two rules that order a move among nodes and modules, the canary (3)
// and the providers (5), and keeps those without which there is nothing to want: the build done, not held, the grant and the
// composition.
func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
switch {
case !m.Built:
@@ -113,6 +112,11 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
}
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return false, delivery.RuleProviders, why
}
}
}
if !m.Granted {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
@@ -153,17 +157,6 @@ func providedHealthy(provision, node string, n Nodes) (bool, string) {
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
}
// unreached is why m on node may be wanted and not reached: the first provision it requires that no provider runs
// healthy on node or at its canary (rule 5, said and never holding a want; issue 1960); empty when all do.
func unreached(m Module, node string, n Nodes) string {
for _, provision := range m.Manifest.Requires {
if ok, why := providedHealthy(provision, node, n); !ok {
return why
}
}
return ""
}
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
// valuable or rebuildable; cache and none do not).
func keepsData(m catalogue.Manifest) bool {
@@ -193,9 +186,6 @@ type Outcome struct {
Restore []string
// Rules is, by node, the rule holding the node back from m.FP and why.
Rules map[string]Holding
// Unreached is, by node, why the module may be wanted there and not reached: no provider of a provision it requires runs
// healthy (rule 5, issue 1960). It holds no want back; the pass says it.
Unreached map[string]string
// Neither is, by node, the nodes the module is assigned to that run it (or whose have names it), hold no want for it after this
// pass and are not held (a held module is on the held list), with the rule holding the first want back: the node's own composition does not hold a module with a fingerprint of the new form, so such a module is on
// neither list and nothing applies it on that node (issue 1945). A module with no fingerprint of the new form goes with the node's
@@ -311,34 +301,27 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
}
ok, rule, words := Holds(mm, node, n)
target := m.FP
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && rule == delivery.RuleCanary {
// The canary orders a move, never the first want of a node that does not run the module (issue 1960: de-spiegel,
// link2pay and photos on novox were left with no want). A node that runs an older fingerprint of the new form,
// composed and granted, gets its first want at what it runs, and rule 3 then orders its move to F as from any want.
// A node that runs an old-form build, which no want can name, stays under the canary (issue 1945, a design limit):
// its first want names F once the canary runs F healthy. Any other node that runs it (F itself, or an older
// fingerprint with no composition or grant) gets F by the rules without which there is nothing to want.
older := ran && runs(have) && have.FP != m.FP && strings.HasPrefix(have.FP, delivery.SourcePrefix2)
if older {
_, composed := m.composition(node, have.FP)
older = composed && m.GrantedAt != nil && m.GrantedAt(have.FP)
}
oldForm := ran && runs(have) && have.FP != m.FP && !strings.HasPrefix(have.FP, delivery.SourcePrefix2)
switch {
case older:
ok, target = true, have.FP
case oldForm:
// held back by rule 3 as written: on neither list until the canary runs F healthy
default:
if have, ran := n.Haves[node][m.Name]; !ok && (!had || before.Removed) && (rule == delivery.RuleCanary || rule == delivery.RuleProviders) && runs(have) {
// A node that runs the module and has no want for it has no want the ordering rules could hold it back to: holding its first
// want back leaves the module on neither list, since the node's own composition gives up every module with a fingerprint of
// the new form (issue 1945: on g14 the bundles of clipmenu, screen-lock and every module requiring x11-display were removed
// and never made again, because xorg's health is unknown and screen-lock's canary never ran it healthy). Its first want names
// what it runs, never a newer fingerprint, so the canary is never passed over: F when it runs F (by the rules without which
// there is nothing to want), or an older fingerprint of the new form whose composition and grant exist. Rules 3 and 5 then
// order the move from it as from any want.
switch _, composed := m.composition(node, have.FP); {
case !ran:
case have.FP == m.FP:
ok, rule, words = holds(mm, node, n, false)
case !strings.HasPrefix(have.FP, delivery.SourcePrefix2):
// An old-form build (not src2:) has no composition, so no want can ever name it: its first want names F, and only by every
// rule as usual, the canary (3) and the providers (5) included, which here hold it back. Nothing is written and the module
// stays on neither list until the canary runs F healthy; then Holds passes and the want names F (issue 1945, a design
// limit set by the lead: the canary stays in force and old-form modules leave neither list by themselves).
case composed && m.GrantedAt != nil && m.GrantedAt(have.FP):
ok, target = true, have.FP
}
}
if why := unreached(m, node, n); why != "" {
if out.Unreached == nil {
out.Unreached = map[string]string{}
}
out.Unreached[node] = why
}
if ok && target == m.FP && len(out.NodeFailed) > 0 && !(had && !before.Removed && before.FP == m.FP) &&
n.Haves[node][m.Name].FP != m.FP {
// While a node fails F alone, F reaches no further node: a node that neither wants nor runs F gets no new want
@@ -65,6 +65,9 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
delivery.RuleCanary: func(_ *Module, n *Nodes) {
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
},
delivery.RuleProviders: func(_ *Module, n *Nodes) {
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
},
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
delivery.RuleComposed: func(m *Module, _ *Nodes) {
m.Composed = func(string, string) (delivery.ComposedDigest, bool) { return delivery.ComposedDigest{}, false }
@@ -83,50 +86,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
}
}
// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
// healthy held photos, and novox offline left a module with no canary.
for _, c := range []struct {
name, module, provision, provider string
have *delivery.Have // what novox runs of the module; nil: a new assignment
before map[string]delivery.Want
online bool
want string
}{
{"de-spiegel, route-proxy down, old form", "de-spiegel", "route", "route-proxy",
&delivery.Have{FP: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}, nil, true, fNew},
{"link2pay, route-proxy down, new assignment", "link2pay", "route", "route-proxy", nil, nil, true, fNew},
{"photos, minio not healthy, moving its want", "photos", "s3-bucket", "minio",
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, true, fNew},
{"novox offline, no canary, new assignment", "link2pay", "route", "route-proxy", nil, nil, false, fNew},
{"novox offline, no canary, moving its want", "photos", "s3-bucket", "minio",
&delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK},
map[string]delivery.Want{"novox": {FP: fOld, Composed: dig(fOld)}}, false, fOld},
} {
m := Module{Name: c.module, FP: fNew, Built: true, Granted: true, Composed: composedEverywhere, Nodes: []string{"novox"},
Manifest: catalogue.Manifest{Module: c.module, Requires: []string{c.provision}}}
n := Nodes{Online: map[string]bool{"novox": c.online}, Haves: map[string]map[string]delivery.Have{"novox": {
c.provider: {FP: fpOf('p'), Phase: delivery.PhaseFailed, Health: delivery.HealthFailed}}},
Providers: map[string]map[string][]string{c.provision: {c.provider: {"novox"}}}}
if c.have != nil {
n.Haves["novox"][c.module] = *c.have
}
out := Wants(m, n, c.before, nil)
w, written := out.Wants["novox"]
if !written {
w = c.before["novox"]
}
if w.FP != c.want || w.Removed {
t.Errorf("%s: novox's want = %+v, want %s", c.name, w, short(c.want))
}
if _, on := out.Neither["novox"]; on {
t.Errorf("%s: on neither list: %+v", c.name, out.Neither)
}
if c.online && !strings.Contains(out.Unreached["novox"], c.provision) {
t.Errorf("%s: the provider down is not said: %q", c.name, out.Unreached["novox"])
}
}
// 1126/AC-SW1, AC-SW2, AC-SW3: a want written Self while g14 was placed, kept by a rule after the placement is gone, names
// the controller's composition of the fingerprint the rule keeps (on 2026-10-27 g14's systemd-resolved, held, and forticlient,
@@ -149,7 +108,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
{name: "rule 5 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}},
}, want: delivery.Want{FP: fNew, Composed: dig(fNew)}, rule: delivery.RuleProviders},
{name: "rule 3 at F", before: fNew, set: func(m *Module, n *Nodes) {
atF(m, n)
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
@@ -1003,12 +962,10 @@ func x11() (Module, Nodes) {
func TestANodeRunningAModuleWithNoWantGetsItsFirstWantAtWhatItRuns(t *testing.T) {
m, n := x11()
out := Wants(m, n, nil, nil)
// 1960: the providers rule holds no want: the canary g14 gets F, its provider's health unknown said
if w := out.Wants["g14"]; w.FP != fNew || out.Unreached["g14"] == "" {
t.Errorf("g14's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["g14"])
}
if w := out.Wants["shanks"]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule != delivery.RuleCanary {
t.Errorf("shanks's want = %+v, want its first want at the fingerprint it runs, held back from F by the canary", w)
for _, node := range []string{"g14", "shanks"} {
if w := out.Wants[node]; w.FP != fOld || w.Composed != dig(fOld) || w.Newest != fNew || w.Rule == 0 {
t.Errorf("%s's want = %+v, want its first want at the fingerprint it runs, held back from F", node, w)
}
}
if len(out.Neither) != 0 {
t.Errorf("left on neither list: %+v", out.Neither)
@@ -1046,22 +1003,11 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
m, n := x11()
change(&m, &n)
out := Wants(m, n, nil, nil)
if what == "an old-form build" {
// the canary g14 does not run F healthy: no want, on neither list by rule 3 (issue 1945)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("%s: shanks got a want: %+v", what, w)
}
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
t.Errorf("%s: shanks on neither list = %+v (named %v), want rule 3", what, h, ok)
}
continue
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("%s: shanks got a want: %+v", what, w)
}
// 1960: no want can name what it runs, so its first want names F rather than none: never on neither list
if w := out.Wants["shanks"]; w.FP != fNew {
t.Errorf("%s: shanks's want = %+v, want F", what, w)
}
if h, ok := out.Neither["shanks"]; ok {
t.Errorf("%s: shanks on neither list = %+v", what, h)
if h, ok := out.Neither["shanks"]; !ok || h.Rule == 0 {
t.Errorf("%s: shanks on neither list = %+v (named %v)", what, h, ok)
}
}
m, n := x11()
@@ -1070,12 +1016,10 @@ func TestNoFirstWantWithoutTheCompositionAndGrantOfWhatItRuns(t *testing.T) {
if len(out.Neither) != 0 {
t.Errorf("nodes holding a want named on neither list: %+v", out.Neither)
}
// the canary g14 moves to F (its provider's health no rule); shanks waits for it
if w := out.Wants["shanks"]; w.FP != "" && w.FP != fOld {
t.Errorf("shanks's want moved from its want past the canary: %+v", w)
}
if w := out.Wants["g14"]; w.FP != fNew {
t.Errorf("the canary g14's want = %+v, want F", w)
for node, w := range out.Wants {
if w.FP != fOld {
t.Errorf("%s's want moved from its want past the ordering rules: %+v", node, w)
}
}
}
@@ -1086,12 +1030,11 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
m, n := x11()
delete(n.Haves["shanks"], "screen-lock")
out := Wants(m, n, nil, nil)
// 1960: the canary orders moves, never a first want: the new assignment is wanted at F
if w := out.Wants["shanks"]; w.FP != fNew {
t.Errorf("a new assignment while the canary does not run F: %+v, want F", w)
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("a new assignment waiting for the canary got a want: %+v", w)
}
if _, ok := out.Neither["shanks"]; ok {
t.Errorf("a new assignment is named on neither list: %+v", out.Neither)
t.Errorf("a new assignment waiting for the canary is named on neither list: %+v", out.Neither)
}
m.Built = false
if out := Wants(m, n, nil, nil); len(out.Neither) != 1 || out.Neither["g14"].Rule != delivery.RuleBuildDone {
@@ -1133,7 +1076,7 @@ func TestANewAssignmentWaitingIsNotOnNeitherList(t *testing.T) {
// 1945/AC21: a node whose have names an old-form build (no fingerprint of the new form, so no composition of it) can never be
// named in a want at what it runs; its first want names F, under the canary rule as usual and with everything F needs and the
// providers rule. Given the canary runs F healthy, shanks's first want names F; given it does not yet, shanks gets no want and
// stays on neither list; the providers hold no want (issue 1960). The old-form node that is itself the canary gets F once its providers run healthy.
// stays on neither list. The old-form node that is itself the canary gets F once its providers run healthy.
func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
oldForm := func(n Nodes, node string) {
n.Haves[node]["screen-lock"] = delivery.Have{FP: "build-1792260876956849831", Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}
@@ -1167,16 +1110,16 @@ func TestAnOldFormNodeGetsItsFirstWantAtFOnlyUnderTheCanary(t *testing.T) {
if h, ok := out.Neither["shanks"]; !ok || h.Rule != delivery.RuleCanary {
t.Errorf("canary not healthy on F: shanks on neither list = %+v (named %v), want rule 3", h, ok)
}
// 1960: the canary healthy on F but the providers not healthy on shanks: F all the same, the provider down said
// the canary healthy on F but the providers not healthy on shanks: rule 5 still holds it back
m, n = x11()
oldForm(n, "shanks")
n.Haves["g14"]["screen-lock"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, nil, nil)
if w := out.Wants["shanks"]; w.FP != fNew || out.Unreached["shanks"] == "" {
t.Errorf("providers not healthy: shanks's want = %+v (unreached %q), want F, the provider said", w, out.Unreached["shanks"])
if w, ok := out.Wants["shanks"]; ok {
t.Errorf("providers not healthy: shanks got a want: %+v", w)
}
if _, on := out.Neither["shanks"]; on {
t.Errorf("providers not healthy: shanks on neither list: %+v", out.Neither)
if h := out.Neither["shanks"]; h.Rule != delivery.RuleProviders {
t.Errorf("providers not healthy: shanks on neither list = %+v, want rule 5", h)
}
// F not granted: nothing, even with the canary healthy
m, n = x11()
@@ -7,7 +7,6 @@ import (
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func load(t *testing.T) catalogue.Manifest {
@@ -33,9 +32,9 @@ func TestManifest(t *testing.T) {
}
reads := slices.Clone(m.Reads)
slices.Sort(reads)
want := []string{"mesh-build-go.steps", "mesh-config.resolved", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants", "mesh-controller.lease",
want := []string{"mesh-build-go.steps", "mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.composed", "mesh-controller.grants",
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "node-engine.have"}
"mesh-delivery-indexer.modules", "mesh-delivery-indexer.placement", "mesh-delivery-indexer.placement-candidates", "mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
t.Errorf("reads = %v, want exactly %v", reads, want)
}
@@ -43,8 +42,7 @@ func TestManifest(t *testing.T) {
for _, s := range m.State {
states[s.Name] = s.History
}
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20,
"placement": 64, "proposals": 5, "candidates": 5, "pulses": 0}; !reflect.DeepEqual(states, want) {
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "puts": 20, "wants": 20}; !reflect.DeepEqual(states, want) {
t.Errorf("state = %v, want %v", states, want)
}
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
@@ -59,27 +57,12 @@ func TestManifest(t *testing.T) {
t.Errorf("uses = %v, want exactly node-build-go and operator-channel", m.Uses)
}
// ADR 0546: it serves its own tools (put, release, puts) from its bundle through the tool runner (the artifact loads the
// module), says each put's step as the event put, and calls the controller's warranted and the bus node's backup holder.
// ADR 0567 §1, §6-§10: it holds the seat mesh-placement, serving its seven verbs; keeps the seat's events on its stream
// placement for 30 days; calls the controller's export, its four verbs that keep its assignments in step and its
// hand-act log; and reads mesh-config's resolved values to compose every node a proposal touches.
if len(m.Claims) != 1 || m.Claims[0].Name != catalogue.PlacementSeat || m.Claims[0].At() != catalogue.ScopeMesh ||
!slices.Equal(m.Claims[0].Serves, []string{"placement", "proposals", "propose", "withdraw", "again", "adopt", "hand-back"}) {
t.Errorf("the assigner claims %+v, want mesh-placement with its seven verbs", m.Claims)
// module), claims no seat (a new seat needs a change of the controller), says each put's step as the event put, and
// calls the controller's warranted and the bus node's backup holder.
if len(m.Claims) != 0 || !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
t.Errorf("the assigner claims %v and serves tools %v, want no claim and its three put tools", m.Claims, m.Tools)
}
if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil {
t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat))
}
if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) {
t.Errorf("the assigner serves tools %v, want its three put tools", m.Tools)
}
if len(m.Streams) != 1 || m.Streams[0].Name != delivery.PlacementStream || m.Streams[0].MaxAgeDays != delivery.PlacementStreamDays ||
!slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) {
t.Errorf("streams = %+v, want placement carrying the seat's four events for %d days", m.Streams, delivery.PlacementStreamDays)
}
if want := []string{"seat:mesh-controller.assign", "seat:mesh-controller.hand-act", "seat:mesh-controller.pin",
"seat:mesh-controller.placement-export", "seat:mesh-controller.unassign", "seat:mesh-controller.unpin",
"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
if want := []string{"seat:mesh-controller.warranted", "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
if !slices.Equal(m.Emits, []string{"put"}) {
+4 -42
View File
@@ -2,36 +2,20 @@
"module": "mesh-delivery-assigner",
"version": "1",
"slug": "assigner",
"claims": [
{
"name": "mesh-placement",
"scope": "mesh",
"serves": [
"placement",
"proposals",
"propose",
"withdraw",
"again",
"adopt",
"hand-back"
]
}
],
"reads": [
"mesh-build-go.steps",
"mesh-config.resolved",
"mesh-controller.assignments",
"mesh-controller.builds",
"mesh-controller.composed",
"mesh-controller.grants",
"mesh-controller.lease",
"mesh-delivery-builder.builds",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-indexer.placement",
"mesh-delivery-indexer.placement-candidates",
"mesh-delivery-teller.answers",
"mesh-delivery-watchman.presence",
"mesh-grants.grants",
"node-engine.have"
],
"uses": [
@@ -39,15 +23,9 @@
"operator-channel"
],
"invokes": [
"seat:mesh-controller.assign",
"seat:mesh-controller.hand-act",
"seat:mesh-controller.pin",
"seat:mesh-controller.placement-export",
"seat:mesh-controller.unassign",
"seat:mesh-controller.unpin",
"seat:mesh-controller.warranted",
"seat:node-backup.backed-up",
"seat:node-backup.now"
"seat:node-backup.now",
"seat:node-backup.backed-up"
],
"emits": [
"put"
@@ -58,28 +36,12 @@
"assigner_puts"
],
"state": [
{ "name": "placement", "history": 64 },
{ "name": "proposals", "history": 5 },
{ "name": "candidates", "history": 5 },
{ "name": "pulses", "key-ttl": true },
{ "name": "asks", "history": 5 },
{ "name": "compositions", "history": 5 },
{ "name": "differences", "history": 64 },
{ "name": "puts", "history": 20 },
{ "name": "wants", "history": 20 }
],
"streams": [
{
"name": "placement",
"events": [
"placement-proposed",
"placement-judged",
"placement-changed",
"placement-withdrawn"
],
"max-age-days": 30
}
],
"own-secrets": {
"broker": "${dir:state}/broker"
},
@@ -7,7 +7,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
@@ -25,8 +24,8 @@ import (
type queueAsker func(key string, value []byte) (json.RawMessage, error)
// Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for
// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, now func() time.Time) []stdio.Tool {
// the builder's read.
func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: build.StuckTool,
@@ -58,70 +57,6 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
return map[string]any{"key": key, "asked": true,
"next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil
}},
{Name: build.BuildAgainTool,
Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " +
"fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " +
"again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " +
"is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " +
"again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " +
"differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.",
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name"},
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
"why": map[string]any{"type": "string", "description": "why it is built again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
raw, err := buildAgain(key, value)
if err != nil {
return nil, err
}
r, err := build.DecodeReAsk(raw)
if err != nil {
return nil, err
}
if r.Refused {
return nil, errors.New(r.Answer)
}
if r.Failed != "" {
return nil, errors.New(r.Answer)
}
if r.Revision == 0 && !r.Pending {
return nil, fmt.Errorf("the builder answered neither a new revision nor a refusal (an older builder?): %s", r.Answer)
}
return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
}},
{Name: build.AnnounceAgainTool,
Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name"},
"fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
"why": map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
raw, err := buildAgain(key, value)
if err != nil {
return nil, err
}
r, err := build.DecodeReAsk(raw)
if err != nil {
return nil, err
}
if r.Refused || r.Revision == 0 {
return nil, errors.New(r.Answer)
}
return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
}},
{Name: build.SeatQueueTool,
Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +
@@ -152,17 +87,8 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
// askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer.
func askQueue(s stdio.KeptState, bound time.Duration) queueAsker {
return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool {
a, err := build.DecodeQueueAsk(v)
return err == nil && !a.Answered.IsZero()
})
}
// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most
// bound for a value written after it that answered says is the builder's answer.
func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker {
return func(key string, value []byte) (json.RawMessage, error) {
heard := make(chan json.RawMessage, 1)
answered := make(chan json.RawMessage, 1)
var rev uint64
var mu sync.Mutex
if err := s.Watch(key, func(c stdio.StateChange) error {
@@ -172,34 +98,36 @@ func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration,
if c.Op != "put" || mine == 0 || c.Revision <= mine {
return nil
}
if answered(c.Value) {
if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() {
select {
case heard <- c.Value:
case answered <- c.Value:
default:
}
}
return nil
}); err != nil {
return nil, fmt.Errorf("watching %s.%s: %w", state, key, err)
return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err)
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err)
return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err)
}
mu.Lock()
rev = r
mu.Unlock()
// The watch may have heard the answer before the revision was known: read it once too.
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) {
return e.Value, nil
if e, err := s.Get(key); err == nil && e != nil && e.Revision > r {
if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() {
return e.Value, nil
}
}
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
case v := <-heard:
case v := <-answered:
return v, nil
case <-wait.Done():
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket)
return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket)
}
}
}
@@ -246,11 +174,7 @@ func main() {
_, err := reasks.Put(key, json.RawMessage(value))
return err
}
buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool {
a, err := build.DecodeReAsk(v)
return err == nil && !a.Answered.IsZero()
})
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, time.Now)); err != nil {
if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err)
os.Exit(1)
}
@@ -92,16 +92,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, delivery.AskLanguages, dead, logger.Printf,
buildrun.SayHearing(ctx, hearingKV, logger.Printf))
defer hearing.Stop()
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
var buildModules []string
for _, l := range delivery.AskLanguages {
if m, err := delivery.BuildModuleOf(l); err == nil {
buildModules = append(buildModules, m)
}
}
heard := buildrun.HearEnds(ctx, asks, asks, buildModules, logger.Printf)
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
calls, err := process.Connect(cred, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
@@ -111,7 +102,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
defer calls.Close()
b := &build.Builder{
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}, Bound: bound},
Asks: buildrun.Asker{Publisher: asks, Waiter: buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}, Bound: bound},
Held: heldOnTheBus{js},
// An ask that can no longer be answered is asked again (issue 1945), read at the step bound: a context's bound.
Gone: heldOnTheBus{js},
@@ -153,11 +144,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
// The toolchains first, so a build that starts from the first modules value finds the nodes.
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
// placement is settled first, so the toolchains settle through the gate already knowing it.
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
watch.put = gate.Toolchain
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, b.NodesChanged, logger.Printf)
assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger)
if err != nil {
return err
}
@@ -254,7 +245,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
case err := <-errs:
return err
case err := <-placementErrs:
case err := <-assignErrs:
return err
case err := <-watch.errs:
return err
@@ -18,14 +18,8 @@ import (
// askAgainer is the part of the builder a reasks value asks through.
type askAgainer interface {
AskAgain(key, why string) []string
BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult
}
// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
// longer for the builder's answer.
const buildAgainBound = 15 * time.Second
// reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the
// key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value
// answered already (this builder's own write) or deleted is left.
@@ -41,15 +35,6 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
if r.Answer != "" {
return
}
if r.Mode == build.ModeBuild {
// The answer comes when the claim and the asks are written, so it is made beside the watch, not in it.
go builtAgain(ctx, b, kv, e, r, logf)
return
}
if r.Mode == build.ModeAnnounce {
announcedAgain(ctx, b, kv, e, r, logf)
return
}
asked := b.AskAgain(e.Key(), r.Why)
if len(asked) == 0 {
r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
@@ -89,52 +74,3 @@ func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Ac
}
return nil
}
// builtAgain answers a reasks value of mode build (the tool builder_build_again, issue 1126 note 72): the failed key is
// built again as a new revision, and the value is written back with the revision and the asks put, or why nothing was done.
func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
res := b.BuildAgain(ctx, r.Module, r.FP, r.Why, r.By, buildAgainBound)
switch {
case res.Refused != "":
r.Answer, r.Refused = res.Refused, true
case res.Failed != "":
r.Answer, r.Revision, r.Failed = fmt.Sprintf("the key is written building as revision %d, and the build failed before any ask was put: %s", res.Revision, res.Failed), res.Revision, res.Failed
default:
r.Pending = res.Revision == 0
r.Answer, r.Revision, r.Asks = "built again: the key is written building as revision "+fmt.Sprint(res.Revision)+" and its asks are put", res.Revision, res.Asks
if res.Pending != "" {
r.Answer = res.Pending
if res.Revision != 0 {
r.Answer = "built again: the key is written building as revision " + fmt.Sprint(res.Revision) + "; " + res.Pending
}
}
}
r.Answered = time.Now().UTC()
logf("%s: build again asked by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer)
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
}
}
// announcedAgain answers a reasks value of mode announce (the tool builder_announce_again): the done key is written again
// unchanged, and the value is written back with the new revision, or why nothing was done.
func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
res := b.AnnounceAgain(ctx, r.Module, r.FP, r.Why, r.By)
if res.Refused != "" {
r.Answer, r.Refused = res.Refused, true
} else {
r.Revision, r.Answer = res.Revision, fmt.Sprintf("announced again: the done key is written again, unchanged, as revision %d", res.Revision)
}
r.Answered = time.Now().UTC()
data, err := json.Marshal(r)
if err != nil {
return
}
if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
}
}
@@ -1,73 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"log"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and
// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
// issue 774).
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
set := &buildrun.ToolchainSet{}
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
// head is told why and opens it again then.
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
src.start()
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
t.Errorf("a trunk head with the source refused: %v, want why", err)
}
if src.opened {
t.Error("a refused source counts as opened, so it is never opened again")
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
@@ -0,0 +1,211 @@
package main
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
// toolchain agreement this file repairs.
import (
"context"
"encoding/json"
"fmt"
"log"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
return false
}
if open {
g.standing[key] = since
} else {
delete(g.standing, key)
}
if !g.known {
return false
}
changed := false
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
return changed
}
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
// passed on changed.
func (g *toolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
if err != nil {
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
changed(ctx)
}
})
}()
return errs, nil
}
@@ -0,0 +1,228 @@
package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
func gateReport(t *testing.T, node, version string) (string, []byte) {
t.Helper()
key, err := delivery.ToolchainKey(node, "typescript")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
if err != nil {
t.Fatal(err)
}
return key, value
}
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
key, value := gateReport(t, node, version)
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(said) != 1 {
t.Fatalf("ace's report is said left out once; said %v", said)
}
// ace reports again while unassigned (an old value seen again): still left out.
key, value := gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
// A report deleted from the bucket goes whether its node is assigned or not.
key, _ = gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
t.Fatal("deleting g14's report passed nothing on")
}
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
// head is told why and opens it again then.
src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
src.start()
if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
t.Errorf("a trunk head with the source refused: %v, want why", err)
}
if src.opened {
t.Error("a refused source counts as opened, so it is never opened again")
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,53 +0,0 @@
package build
import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A done build is announced again by hand** (recovery of photos and link2pay on novox, 2026-10-29). The controller takes a
// build in when a write of the build key reaches its take-in consumer, and composes then; a done key is never written again
// by the builder, so a composition or a grant lost afterwards could not be made again by any tool. AnnounceAgain writes the
// done key again with its own bytes, as a new revision of the same key: its content, state, artifacts and manifest stay
// identical, and only the write is new. Who asked and why is in the builder's log and in the reasks answer.
// AnnounceAgain rewrites the done key of module at fp unchanged, as a new revision, and answers the revision. A key that is
// not done is refused with why: failed (builder_build_again builds it) or waiting (builder_ask_again).
func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) BuildAgainResult {
b.init()
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return BuildAgainResult{Refused: err.Error()}
}
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
}
if !found {
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing to announce", key)}
}
cur, err := delivery.DecodeBuilderBuild(value)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
}
switch cur.State {
case delivery.BuildDone:
case delivery.BuildFailed:
return BuildAgainResult{Refused: fmt.Sprintf("%s is failed, not done: %s builds it again", key, BuildAgainTool)}
default:
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not done: %s asks its waiting asks again", key, cur.State, AskAgainTool)}
}
// Compare-and-set at the revision read: a build written meanwhile is not overwritten.
nrev, err := b.Claimer.Store.Update(ctx, key, value, rev)
if err != nil {
if errors.Is(err, ErrConflict) {
return BuildAgainResult{Refused: fmt.Sprintf("%s changed while it was announced: read it again", key)}
}
return BuildAgainResult{Refused: fmt.Sprintf("%s could not be written: %v", key, err)}
}
b.logf("%s: done, announced again by %s (revision %d): %s", key, by, nrev, why)
return BuildAgainResult{Revision: nrev}
}
@@ -1,49 +0,0 @@
package build
import (
"bytes"
"context"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func TestAnnounceAgainRewritesOnlyADoneKeyUnchanged(t *testing.T) { // recovery of photos and link2pay, 2026-10-29
for _, c := range []struct {
name string
key *delivery.BuilderBuild
fp string
refuse string
}{
{name: "a done key is written again, unchanged, as a new revision", key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
{name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "x"}},
{name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked"}},
{name: "a key with no build is refused", refuse: "has no build key"},
{name: "a fingerprint that is not one is refused", refuse: "fingerprint", fp: "nope"},
} {
t.Run(c.name, func(t *testing.T) {
r := newRig(t, goBundle)
var rev uint64
if c.key != nil {
c.key.Manifest = manifestD
rev = putBuild(t, r.st, r.key, *c.key)
}
before := bytes.Clone(r.st.vals[r.key])
fp := fp1
if c.fp != "" {
fp = c.fp
}
res := r.b.AnnounceAgain(context.Background(), "m", fp, "grant lost", "shanks.lead")
if c.refuse != "" {
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || r.st.revs[r.key] != rev || !bytes.Equal(r.st.vals[r.key], before) {
t.Fatalf("answer %+v, key at revision %d (was %d); want refused with %q and nothing written", res, r.st.revs[r.key], rev, c.refuse)
}
return
}
if res.Refused != "" || res.Revision <= rev || r.st.revs[r.key] != res.Revision || !bytes.Equal(r.st.vals[r.key], before) {
t.Fatalf("answer %+v, key at revision %d (was %d); want a new revision and the same bytes", res, r.st.revs[r.key], rev)
}
})
}
}
@@ -1,183 +0,0 @@
package build
import (
"context"
"fmt"
"maps"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again
// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a
// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed
// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key
// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool
// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps
// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused.
// BuildAgainResult is the builder's answer to a request to build a key again.
type BuildAgainResult struct {
// Refused is why nothing was done, "" when the key was written building as a new revision.
Refused string
// Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound.
Revision uint64
Asks map[string]string
// Pending says the builder had not claimed the key, or not put every ask, when it answered: not a refusal.
Pending string
// Failed is the error of the key when the build ended failed before its asks were put (a plan that cannot be read, a
// toolchain not agreed): the new revision was written, and nothing was asked.
Failed string
}
// handWatch follows one request to build a key again until its asks are put.
type handWatch struct {
rev uint64
lanes int
asks map[string]string
done chan struct{}
shut bool
// ended is set when the look ended before the asks were all put.
ended bool
}
// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held.
func (w *handWatch) settle() {
if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes {
w.shut = true
close(w.done)
}
}
func (b *Builder) hand(key string, f func(w *handWatch)) {
b.mu.Lock()
defer b.mu.Unlock()
if w := b.hands[key]; w != nil {
f(w)
}
}
func (b *Builder) handClaimed(key string, rev uint64) {
b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() })
}
func (b *Builder) handLanes(key string, n int) {
b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() })
}
func (b *Builder) handAsk(key, language, id string) {
b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() })
}
// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended.
func (b *Builder) handEnded(key string) {
b.hand(key, func(w *handWatch) {
if !w.shut {
w.shut, w.ended = true, true
close(w.done)
}
})
}
// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a
// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key
// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again).
// ctx is the builder's own context: the build it starts lives on it.
func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult {
b.init()
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return BuildAgainResult{Refused: err.Error()}
}
value, _, found, err := b.Claimer.Store.Get(ctx, key)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
}
if !found {
return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)}
}
cur, err := delivery.DecodeBuilderBuild(value)
if err != nil {
return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
}
switch {
case cur.State == delivery.BuildDone:
return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)}
case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued:
return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)}
case cur.State != delivery.BuildFailed:
return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)}
case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix):
return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)}
}
// One critical section: the key is refused when it is looked at or asked already, the module still has the fingerprint,
// and the watch and the look are registered before anything else can take the key.
w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})}
b.mu.Lock()
m, known := b.known[module]
switch {
case !known || m.FP2 != fp:
b.mu.Unlock()
return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)}
case m.Manifest == "":
b.mu.Unlock()
return BuildAgainResult{Refused: fmt.Sprintf("module %s names no manifest at %s yet, so the builder looks at nothing", module, fp)}
case b.inflight[key] || b.hands[key] != nil:
b.mu.Unlock()
return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now (or asked again by hand): wait for it to end", key)}
}
b.hands[key] = w
b.inflight[key] = true
b.mu.Unlock()
defer func() {
b.mu.Lock()
if b.hands[key] == w {
delete(b.hands, key)
}
b.mu.Unlock()
}()
b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why)
b.startOnce(ctx, key, module, m)
wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8)
defer stop()
select {
case <-w.done:
case <-wait.Done():
}
b.mu.Lock()
res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)}
ended, shut, lanes := w.ended, w.shut, w.lanes
b.mu.Unlock()
switch {
case res.Revision == 0 && shut:
res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key)
case res.Revision == 0:
res.Pending = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound)
case ended && len(res.Asks) < max(lanes, 0)+boolInt(lanes < 0):
// The look ended before every ask was put: say how the key stands, not that it was asked.
if v, _, found, err := b.Claimer.Store.Get(ctx, key); err == nil && found {
if cur, err := delivery.DecodeBuilderBuild(v); err == nil {
switch cur.State {
case delivery.BuildFailed:
res.Failed = cur.Error
case delivery.BuildDone:
default:
res.Pending = fmt.Sprintf("the look ended with the key %s (%s) before every ask was put: its log says why", cur.State, cur.Step)
}
}
}
case !shut:
res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", res.Revision, bound)
}
return res
}
func boolInt(b bool) int {
if b {
return 1
}
return 0
}
@@ -89,8 +89,6 @@ type Builder struct {
held map[string]bool
// waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them.
waiting map[string]map[string]chan string
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
hands map[string]*handWatch
}
type planned struct {
@@ -108,7 +106,6 @@ func (b *Builder) init() {
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
b.hands = map[string]*handWatch{}
})
}
@@ -375,35 +372,9 @@ func (b *Builder) consider(ctx context.Context, module string, m delivery.Module
}()
}
// startOnce looks at the module for a key the caller has set inflight already (BuildAgain): the by-hand look runs once, and
// a look the builder queued for the key meanwhile (again) still runs after it, as consider's loop does.
func (b *Builder) startOnce(ctx context.Context, key, module string, m delivery.Module) {
b.wg.Add(1)
go func() {
defer b.wg.Done()
b.sem <- struct{}{}
defer func() { <-b.sem }()
for {
b.try(ctx, module, m)
b.mu.Lock()
next, more := b.again[key]
delete(b.again, key)
if !more || ctx.Err() != nil {
delete(b.inflight, key)
b.mu.Unlock()
return
}
b.mu.Unlock()
m = next // a look the builder queued meanwhile is not lost (consider's rule); it is not a second by-hand build
}
}()
}
// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
key, _ := delivery.BuilderBuildKey(module, m.FP2)
defer b.handEnded(key)
plan, planErr, read := b.planOf(ctx, m.Manifest)
nothing := read && planErr == nil && len(plan.Tasks) == 0
if planErr == nil && !nothing {
@@ -414,6 +385,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
return
}
}
key, _ := delivery.BuilderBuildKey(module, m.FP2)
b.mu.Lock()
trunk := b.trunkFP[key]
first := !b.tried[key]
@@ -428,7 +400,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
if cl == nil {
return
}
b.handClaimed(key, cl.Rev)
if cl.Unasked != nil {
b.logf("%v", cl.Unasked)
}
@@ -564,9 +535,6 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
}
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
b.handLanes(key, len(lanes))
}
var wg sync.WaitGroup
for _, l := range lanes {
wg.Add(1)
@@ -749,11 +717,7 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error {
v.Asks = map[string]string{}
}
v.Asks[language] = id
if err := n.b.Claimer.Finish(ctx, n.cl, v); err != nil {
return err
}
n.b.handAsk(n.cl.Key, language, id)
return nil
return n.b.Claimer.Finish(ctx, n.cl, v)
}
// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the
@@ -691,129 +691,18 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
byHand bool
always bool // Gone says every ask cannot be answered, not only the first
dead bool // DEAD_LETTERS keeps the first ask: the seat gave it up (issue 734)
deaf bool // the ask's own watch hears nothing though its step ends (the lost check of 2026-10-28)
puts int
done bool
asking int // the asking the event says, 0 when none is said
// again, when set, is a request to build the key again by hand (BuildAgain, issue 1126 note 72) of the key as
// it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it
// is refused, "" when it is built again as a new revision and its ask id answered.
again bool
key *delivery.BuilderBuild
known string
refuse string
// edit changes the module the builder has read, busy marks the key looked at (1) or asked by hand (2), failed is a part of
// the error the key then holds when its build ends before its asks are put.
edit func(m *delivery.Module)
busy int
failed string
// during is a look of the builder's own at the module while the by-hand build runs: it still runs afterwards.
during bool
}{
{name: "build again: a look the builder queued during the by-hand build still runs afterwards", again: true, during: true,
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a key looked at by the builder now is refused, not built twice", again: true, refuse: "is being looked at", busy: 1,
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a key already asked by hand is refused, its watch kept", again: true, refuse: "is being looked at", busy: 2,
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a module with no manifest is refused", again: true, refuse: "names no manifest", edit: func(m *delivery.Module) { m.Manifest = "" },
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a toolchain that moved writes nothing and is said at once", again: true, refuse: "wrote nothing",
edit: func(m *delivery.Module) {
m.Toolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.0", Recipe: "1"}}
},
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a build that fails before its asks says the key failed, not that it was asked", again: true, failed: "cannot be read",
edit: func(m *delivery.Module) { m.Manifest = mdigest('b') },
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "build again: a failed key is written building as a new revision and asks the seat", again: true,
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}},
{name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do",
key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
{name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting",
key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}},
{name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry",
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}},
{name: "build again: a key with no build is refused", again: true, refuse: "has no build key"},
{name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64),
key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}},
{name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2},
{name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2},
{name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2},
{name: "its step ended while its watch was deaf: read at its build module's next step end, done with no new ask", deaf: true, puts: 1, done: true},
{name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2},
{name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0},
} {
t.Run(c.name, func(t *testing.T) { // 1945
r := newRig(t, goBundle)
if c.again {
var rev uint64
if c.key != nil {
c.key.Manifest = manifestD
rev = putBuild(t, r.st, r.key, *c.key)
}
m, err := delivery.DecodeModule(moduleValue(t, "modules/m"))
if err != nil {
t.Fatal(err)
}
if c.known != "" {
m.FP2 = c.known
}
if c.edit != nil {
c.edit(&m)
}
r.b.init()
var stored atomic.Int64
if c.during {
r.asks.Answer = nil
r.b.Stored = func(context.Context, []string) ([]string, error) { stored.Add(1); return nil, nil }
}
switch c.busy {
case 1:
r.b.inflight[r.key] = true
case 2:
r.b.hands[r.key] = &handWatch{}
}
r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked
log := &logged{}
r.b.Logf = log.logf
res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second)
if c.during {
// The trunk's modules value arrives while the by-hand build waits for its ask: it is queued, and run after it.
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || res.Revision <= rev || stored.Load() != 1 {
t.Fatalf("key %+v, answer %+v, %d asks of the store; want built from the new revision and the queued look run once after it", v, res, stored.Load())
}
return
}
r.b.Wait()
after := r.st.revs[r.key]
if c.refuse != "" {
if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || len(r.asks.Asks()) != 0 || after != rev {
t.Fatalf("refused %q (revision %d, %d asks, key at revision %d was %d); want refused with %q and nothing written or asked", res.Refused, res.Revision, len(r.asks.Asks()), after, rev, c.refuse)
}
return
}
if c.failed != "" {
if !strings.Contains(res.Failed, c.failed) || res.Revision <= rev || len(res.Asks) != 0 || len(r.asks.Asks()) != 0 {
t.Fatalf("answer %+v after %d asks; want a new revision (was %d), no ask and the key's failure %q", res, len(r.asks.Asks()), rev, c.failed)
}
return
}
asks := r.asks.Asks()
if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID {
t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev)
}
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision {
t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision)
}
if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") {
t.Errorf("the why is not logged: %v", log.lines)
}
return
}
r.asks.Answer = nil
r.b.Askings = c.askings
var said []AskedAgain
@@ -831,12 +720,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
if c.dead {
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: buildrun.LostAskWaiter{Inner: r.asks, Dead: dead}, Bound: 2 * time.Second}
}
var heard *buildrun.HeardEnds
if c.deaf {
heard = &buildrun.HeardEnds{Inner: silentWatch{}, Read: r.asks}
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: heard, Bound: 2 * time.Second}
}
if !c.byHand && !c.dead && !c.deaf {
if !c.byHand && !c.dead {
gone.why = func(a delivery.StepAsk) string {
if c.always || a.ID == first || first == "" {
return "it waits first in line while a node took a later ask"
@@ -858,16 +742,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
}
}
}
if c.deaf {
r.asks.End(r.answer(waitForAsks(t, r, 1)[0]))
for deadline := time.Now().Add(2 * time.Second); heard.Waiting() == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
t.Fatal("no ask is waited on")
}
}
heard.Nudge(buildrun.GoBuildModule) // a step of mesh-build-go ended
}
if c.done && !c.deaf {
if c.done {
asks := waitForAsks(t, r, c.puts)
if first == "" && !c.byHand {
first = asks[0].ID
@@ -892,7 +767,7 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
mu.Lock()
defer mu.Unlock()
// Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream.
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && !c.deaf && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
if gone.mu.Lock(); c.done && !c.byHand && !c.dead && (len(gone.dropped) != 1 || gone.dropped[0] != first) {
t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first)
}
gone.mu.Unlock()
@@ -1102,10 +977,3 @@ func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) {
t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks()))
}
}
// silentWatch is a watch of an ask's step that stays open and hears nothing.
type silentWatch struct{}
func (silentWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) {
return make(chan delivery.Step), nil
}
@@ -22,11 +22,6 @@ import (
const (
StuckTool = "builder_stuck"
AskAgainTool = "builder_ask_again"
// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
BuildAgainTool = "builder_build_again"
// AnnounceAgainTool writes a done key again unchanged, so the controller takes it in and composes again: a reasks value
// of Mode ModeAnnounce.
AnnounceAgainTool = "builder_announce_again"
// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
ReAsksState = "reasks"
ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
@@ -41,27 +36,8 @@ type ReAsk struct {
At time.Time `json:"at"`
Answer string `json:"answer,omitempty"`
Answered time.Time `json:"answered,omitzero"`
// Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again.
Mode string `json:"mode,omitempty"`
Module string `json:"module,omitempty"`
FP string `json:"fp,omitempty"`
// Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new
// revision (building) and the ask ids it put by language.
Refused bool `json:"refused,omitempty"`
// Pending says the builder answered before the key was claimed or all its asks were put: neither a refusal nor done.
// Failed is the error of a key written building that failed before its asks were put.
Pending bool `json:"pending,omitempty"`
Failed string `json:"failed,omitempty"`
Revision uint64 `json:"revision,omitempty"`
Asks map[string]string `json:"asks,omitempty"`
}
// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
const ModeBuild = "build"
// ModeAnnounce is the Mode of a reasks value that asks a done key to be announced again.
const ModeAnnounce = "announce"
// DecodeReAsk reads a reasks value.
func DecodeReAsk(data []byte) (ReAsk, error) {
var r ReAsk
@@ -87,32 +63,6 @@ func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error)
return key, data, err
}
// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value.
func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return "", nil, err
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why the key is built again (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp})
return key, data, err
}
// NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value.
func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return "", nil, err
}
if strings.TrimSpace(why) == "" {
return "", nil, errors.New("say why the done build is announced again (why)")
}
data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeAnnounce, Module: module, FP: fp})
return key, data, err
}
// StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
type StateRead struct {
Keys func() ([]string, error)
@@ -92,8 +92,8 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
}
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains",
"mesh-delivery-assigner.placement", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
"mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
@@ -116,7 +116,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {
+1 -3
View File
@@ -3,9 +3,7 @@
"version": "1",
"slug": "builder",
"tools": [
"builder_announce_again",
"builder_ask_again",
"builder_build_again",
"builder_seat_queue",
"builder_stuck"
],
@@ -33,7 +31,7 @@
"mesh-build-image.steps",
"mesh-build-node.steps",
"mesh-build-node.toolchains",
"mesh-delivery-assigner.placement",
"mesh-controller.assignments",
"mesh-delivery-gate.source",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
@@ -1,6 +1,6 @@
// Command mesh-delivery-checker-tools serves the checker's read tools (novox/hq ADR 0484 clause 13, issue 1686): the runtime
// launches it for a call, it reads the checker's verdicts and kept states and the indexer's candidates through the checker's
// state and reads. It writes only the tries (the try's tools) and, asking a check again by hand, one verdict. The checker is the other program: it serves no tool (ADR 0437 §4).
// state and reads, and writes nothing. The checker is the other program: it serves no tool (ADR 0437 §4).
package main
import (
@@ -250,24 +250,6 @@ func Tools(r check.Reader) []stdio.Tool {
}
}
// RecoveryTools are the tools that steer a check by hand (tools-are-how-a-module-is-spoken-to): ask a check again whose
// wait judged nothing (the lost check of 2026-10-28). They write the checker's verdicts through w.
func RecoveryTools(r check.Reader, w check.VerdictEntry) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
return []stdio.Tool{{Name: check.AskAgainTool,
Description: "Ask a candidate's check again by hand, whose verdict stays running while nothing runs for it (" + check.StatusReadTool + " names it " +
"under lost): its verdict is ended as asked again, saying why, and the checker, which hears its verdicts, stops the wait that judged nothing " +
"and asks the check anew, whole. Refused for a group with no candidate, a verdict that is not running (a pass or failure stands until a push or " +
"a trunk move), and while a step of its pull requests can still run on a build module (its end judges it). The checker reads a step that " +
"ended while its watch was deaf by itself at the next step end of its build module; this is for what that does not reach.",
Input: map[string]any{"type": "object", "required": []string{"group", "why"}, "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"},
"why": map[string]any{"type": "string", "description": "why it is asked again, in words; kept in the verdict it ends"}}},
Run: func(args map[string]any) (any, error) {
return r.AskAgain(w, str(args, "group"), str(args, "why"))
}}}
}
// stepBound is the checker's step bound as the checker reads it (MESH_CHECKER_STEP_BOUND), its default when unset or not a
// duration.
func stepBound() time.Duration {
@@ -282,8 +264,7 @@ func main() {
PreviewState: read(stdio.State("previews")),
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)),
AskState: read(stdio.State(check.AsksState))}
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState))}
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
tries := stdio.State(check.TriesState)
source := stdio.State("mesh-delivery-gate.source")
@@ -311,20 +292,7 @@ func main() {
return err
}}
trying := Trying{Tries: store, Ender: store, Trunks: trunks}
verdicts := stdio.State("verdicts")
written := stateStore{entry: func(key string) (json.RawMessage, uint64, error) {
reads.Add(1)
e, err := verdicts.Get(key)
if err != nil || e == nil {
return nil, 0, err
}
return e.Value, e.Revision, nil
}, update: func(key string, value []byte, revision uint64) error {
_, err := verdicts.Update(key, json.RawMessage(value), revision)
return err
}}
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
if err := stdio.Serve("", logged(tools)); err != nil {
if err := stdio.Serve("", logged(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...))); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
}
@@ -23,14 +23,6 @@ func TestTheToolsServedAreTheFiveReadTools(t *testing.T) {
if !slices.Equal(names, want) {
t.Errorf("tools = %v, want %v", names, want)
}
// The recovery tool beside them asks a lost check again; with nothing to write it refuses.
recovery := RecoveryTools(check.Reader{}, nil)
if len(recovery) != 1 || recovery[0].Name != check.AskAgainTool || recovery[0].Description == "" {
t.Errorf("recovery tools = %+v", recovery)
}
if _, err := recovery[0].Run(map[string]any{"group": "feat/x", "why": "lost"}); err == nil {
t.Errorf("%s answered with no state to write", check.AskAgainTool)
}
// With nothing to read from a tool says so and changes nothing.
for _, tl := range Tools(check.Reader{}) {
if tl.Name == check.StatusReadTool {
@@ -9,7 +9,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
)
@@ -131,55 +130,3 @@ func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.
func parseManifest(raw string) (catalogue.Manifest, error) {
return catalogue.ParseManifestForInputs([]byte(raw))
}
// awaitedKV is a bucket that may appear after the start (process.AwaitBucket) as a check.KV: a call while it is not on the
// bus fails saying so.
type awaitedKV struct{ b *process.AwaitedBucket }
func (a awaitedKV) kv() (kvStore, error) {
kv := a.b.KV()
if kv == nil {
return kvStore{}, errors.New("the bucket is not on the bus yet")
}
return kvStore{kv}, nil
}
func (a awaitedKV) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
s, err := a.kv()
if err != nil {
return nil, 0, false, err
}
return s.Get(ctx, key)
}
func (a awaitedKV) Create(ctx context.Context, key string, value []byte) (uint64, error) {
s, err := a.kv()
if err != nil {
return 0, err
}
return s.Create(ctx, key, value)
}
func (a awaitedKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
s, err := a.kv()
if err != nil {
return 0, err
}
return s.Update(ctx, key, value, revision)
}
func (a awaitedKV) Keys(ctx context.Context) ([]string, error) {
s, err := a.kv()
if err != nil {
return nil, err
}
return s.Keys(ctx)
}
func (a awaitedKV) Delete(ctx context.Context, key string) error {
s, err := a.kv()
if err != nil {
return err
}
return s.Delete(ctx, key)
}
@@ -89,8 +89,6 @@ type fakeBuildModules struct {
// beside is, by workspace, the siblings the fetch placed in it as `<owner/repo> <commit>`, sorted: a digest echoes them
// (1929/AC2; mesh-sdk buildstep.besideOf), or the level runs whole and keeps nothing.
beside map[string][]string
// names are the nodes' names the last check ask carried, for the names check (ADR 0567 §12).
names []string
}
func (f *fakeBuildModules) count(tool string) int {
@@ -193,14 +191,7 @@ func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.Key
}
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: "n1", Put: put,
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) {
if a.Check != nil {
f.mu.Lock()
f.names = a.Check.NodeNames
f.mu.Unlock()
}
return buildstep.RunAsk(ctx, steps, a)
}}
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
go func() { _ = p.Pull(f.ctx) }()
}
@@ -343,7 +334,6 @@ func startBed(t *testing.T) *bed {
t.Fatal(err)
}
b.setOnline(t, "n1", true)
b.setNode(t, "n1")
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
b.setHealth(t, "n1", module)
}
@@ -376,20 +366,6 @@ func (b *bed) setOnline(t *testing.T, node string, online bool) {
}
}
// setNode writes the node's record in the node-engine's have, as the node-engine does, so the checker names it in each
// check ask (ADR 0567 §12); with no record the checker asks no check.
func (b *bed) setNode(t *testing.T, node string) {
t.Helper()
data, err := delivery.EncodeNodeRecord(delivery.NodeRecord{V: 1, Node: node})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveNodeKey(node)
if _, err := b.buckets[delivery.HaveBucket].Put(b.ctx, key, data); err != nil {
t.Fatal(err)
}
}
func (b *bed) setHealth(t *testing.T, node, module string) {
t.Helper()
data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
@@ -517,12 +493,6 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || len(v.Machines) != 1 || v.Machines[0] != "n1" {
t.Errorf("composition %+v machines %v", v.Composition, v.Machines)
}
b.fake.mu.Lock()
names := b.fake.names
b.fake.mu.Unlock()
if len(names) != 1 || names[0] != "n1" {
t.Errorf("the check ask named the nodes %v, want [n1] from the have's node record", names)
}
o := b.owed(t, 7, head1, "success")
if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run") {
t.Errorf("owed = %+v", o)
@@ -827,8 +797,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
break
}
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") ||
!said.Keeps("asks") || said.Readable("asks") || len(said.States) != 7 {
t.Fatalf("the checker said %+v, want its seven states, asks and hearing its own, with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || !said.Readable("tries") || said.Readable("hearing") || len(said.States) != 6 {
t.Fatalf("the checker said %+v, want its six states with kept, owed, previews, tries and verdicts readable, tries by the delivery view (1929/AC4; 1887 A5b; build_usage)", said)
}
}
@@ -2,16 +2,12 @@ package main
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
)
@@ -29,46 +25,11 @@ type judgeDeps struct {
builds check.BuildStates
// compositions is the assigner's bucket of compositions; nil leaves every composition not run.
compositions check.CompositionReader
// nodes names every node with a record; nil hands no names.
nodes *nodeNames
logf func(format string, args ...any)
}
// nodeNames reads every node's name from the node-engine's have, one key `<node>.node` per node with a record (novox/hq
// ADR 0567 §12), at each ask: a node enrolled since is named in the next check.
type nodeNames struct{ kv jetstream.KeyValue }
// openNodeNames opens the node-engine's have for the nodes' names alone: it is not watched, so no check waits on a node.
func openNodeNames(ctx context.Context, js process.BucketOpener, logf func(string, ...any)) *nodeNames {
return &nodeNames{kv: process.OptionalBucket(ctx, js, delivery.HaveBucket, "no check is asked until it is: the names check needs the nodes' names", logf)}
}
func (n nodeNames) Names(ctx context.Context) ([]string, error) {
if n.kv == nil {
return nil, fmt.Errorf("the bucket %s could not be opened at the checker's start", delivery.HaveBucket)
}
lister, err := n.kv.ListKeysFiltered(ctx, "*."+delivery.HaveNodeWord)
if err != nil {
return nil, err
}
var out []string
for key := range lister.Keys() {
if node, _, isNode, err := delivery.ParseHaveKey(key); err == nil && isNode {
out = append(out, node)
}
}
sort.Strings(out)
return out, nil
logf func(format string, args ...any)
}
func newJudge(d judgeDeps) check.Judge {
var nodes interface {
Names(ctx context.Context) ([]string, error)
}
if d.nodes != nil {
nodes = d.nodes
}
return &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
return &check.Examiner{Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf}
}
@@ -69,8 +69,6 @@ var keptStates = []announce.State{
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
// Whether DEAD_LETTERS is heard (issue 734): read by the checker's own status tool alone.
{Name: "hearing"},
// The asks each running check put (check.AsksState): read by the checker's own checker_ask_again alone.
{Name: "asks"},
{Name: "kept", Readable: true},
{Name: "owed", Readable: true},
{Name: "previews", Readable: true},
@@ -146,14 +144,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
hearing := buildrun.NewDeadLetterHearing(ctx, buildrun.JetStreamDeadLetters{JS: js}, []string{delivery.AskGo, delivery.AskNode}, dead,
logger.Printf, buildrun.SayHearing(ctx, hearingKV, logger.Printf))
defer hearing.Stop()
// Each ask's step is heard by its own watch and, at every step end of its build module and every reconnect, read
// (buildrun.HearEnds): a watch that goes deaf loses no end (the lost check of 2026-10-28).
heard := buildrun.HearEnds(ctx, asks, asks, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, logger.Printf)
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again(); heard.Nudge("") })
waiter := buildrun.LostAskWaiter{Inner: heard, Dead: dead, Hearing: hearing}
asksKV := process.AwaitBucket(ctx, js, conn, check.AsksBucket, "checker_ask_again refuses every check, as it cannot tell whether its ask waits in a stream", logger.Printf)
conn.SetReconnectHandler(func(*nats.Conn) { hearing.Again() })
waiter := buildrun.LostAskWaiter{Inner: asks, Dead: dead, Hearing: hearing}
ch := &check.Checker{
Asks: awaitedKV{asksKV},
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
Trunks: trunks,
@@ -201,18 +194,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce, compositions: compositions,
nodes: openNodeNames(ctx, js, logger.Printf),
})
ch.Announce = announce
tryer.Judge = ch.Judge
// The toolchains first, so a check that starts from the first candidate finds the nodes.
watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger)
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
// placement is settled first, so the toolchains settle through the gate already knowing it.
gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
watch.put = gate.Toolchain
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, ch.NodesChanged, logger.Printf)
assignErrs, err := watchAssignments(ctx, js, gate, ch.NodesChanged, logger)
if err != nil {
return err
}
@@ -338,21 +330,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
})
}()
// The checker's own verdicts, updates only, in one watch: one checker_ask_again wrote asks its check again
// (Checker.VerdictWritten); a deleted key of heads a candidate holds is judged again (Checker.VerdictDeleted, issue 1733).
vw, err := buckets[delivery.CheckerVerdictsBucket].WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
return fmt.Errorf("the checker's verdicts cannot be watched, so no check can be asked again by hand and a deleted verdict would never be written again: %w", err)
}
go func() {
errs <- follow(ctx, vw, func(e jetstream.KeyValueEntry) {
if opOf(e) {
ch.VerdictWritten(ctx, e.Key(), e.Value())
} else {
ch.VerdictDeleted(ctx, e.Key())
}
})
}()
logger.Printf("started on %s", cred.Node)
go func() { errs <- follow(ctx, w, each) }()
select {
@@ -362,7 +339,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return nil
case err := <-errs:
return err
case err := <-placementErrs:
case err := <-assignErrs:
return err
case err := <-watch.errs:
return err
@@ -7,8 +7,7 @@ import (
)
// 1912: with checks as asks on the build seats' streams, the stream redelivers a lost node's ask (ADR 0525 §6), so the
// checker program watches no node-engine have to start again a check that failed for want of a node. It reads the have's
// keys only for the nodes' names at each ask (judge.go, novox/hq ADR 0567 §12).
// checker program opens no node-engine have to start again a check that failed for want of a node.
func TestTheCheckerWatchesNoHave(t *testing.T) {
raw, err := os.ReadFile("main.go")
if err != nil {
@@ -1,62 +0,0 @@
package main
import (
"bytes"
"context"
"errors"
"log"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// 1945 (a part exits at start for a bucket it works without): a bus that refuses every toolchains bucket leaves the
// watch unstarted and the part running, said; a refused bucket it cannot work without is an error naming the bucket and
// its grant. The gate over the seat's placement, and its refusal, are mesh-sdk's (buildrun.ToolchainGate,
// issue 774).
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
set := &buildrun.ToolchainSet{}
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-build-node_toolchains")}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
@@ -0,0 +1,211 @@
package main
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
// toolchain agreement this file repairs.
import (
"context"
"encoding/json"
"fmt"
"log"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
return false
}
if open {
g.standing[key] = since
} else {
delete(g.standing, key)
}
if !g.known {
return false
}
changed := false
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
return changed
}
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
// passed on changed.
func (g *toolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
if err != nil {
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
changed(ctx)
}
})
}()
return errs, nil
}
@@ -0,0 +1,218 @@
package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
func gateReport(t *testing.T, node, version string) (string, []byte) {
t.Helper()
key, err := delivery.ToolchainKey(node, "typescript")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
if err != nil {
t.Fatal(err)
}
return key, value
}
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
key, value := gateReport(t, node, version)
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(said) != 1 {
t.Fatalf("ace's report is said left out once; said %v", said)
}
// ace reports again while unassigned (an old value seen again): still left out.
key, value := gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
// A report deleted from the bucket goes whether its node is assigned or not.
key, _ = gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
t.Fatal("deleting g14's report passed nothing on")
}
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
// is an error naming the bucket and its grant.
var logs bytes.Buffer
logger := log.New(&logs, "", 0)
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
t.Errorf("a refused watch of the assignments stopped the start: %v", err)
}
if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
t.Errorf("a refused toolchains bucket stopped the start: %v", err)
}
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
t.Errorf("the refusals were not said: %q", logs.String())
}
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "checker", "mesh-delivery-indexer_modules", "work"); err == nil ||
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
}
}
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
refuseOpen bool
err error
}
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
if r.refuseOpen {
return nil, r.err
}
return refusedWatch{err: r.err}, nil
}
type refusedWatch struct {
jetstream.KeyValue
err error
}
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
return nil, r.err
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027220144-3f7957e16972
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027220144-3f7957e16972 h1:iYDYv4HpV1ic20Ban0MaOErryQfNT8ngjlh0bKW6sCU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027220144-3f7957e16972/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -1,198 +0,0 @@
package check
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A check is asked again by hand** (the lost check of 2026-10-28; tools-are-how-a-module-is-spoken-to: every recovery
// has a tool). checker_ask_again ends a candidate's running verdict, whose wait judged nothing, with AskedAgainWords and
// the why; the checker hears it on its watch of its verdicts (Checker.VerdictWritten), ends the check of those heads
// running in it, and claims and asks it anew, a whole check (IsEnded). It is refused while a step of the candidate's pull
// requests can still run (checker_status's running_steps): that step's end judges it; and while an ask the check put is
// still in its build seat's stream, taken by no node (it has no step key yet): asking again then would run the check twice
// (issue 1791's kind). The checker keeps the asks each running check put in its state asks (AsksState, by verdict key).
// An ask is not put again under its own id instead: the stream de-duplicates an id within its window, so the copy waiting
// stays and the new one is dropped, and past the window both would run; nothing takes a put ask back (buildrun.Asker).
// AsksState is the checker's state of the asks each running check put, by verdict key (CheckAsks); AsksBucket its bucket.
const (
AsksState = "asks"
AsksBucket = delivery.CheckerModule + "_" + AsksState
)
// CheckAsks are the asks a check put, in order, as the state asks keeps them while it runs; its end deletes the key.
type CheckAsks struct {
Asks []string `json:"asks"`
}
// AsksKV is the state asks: a KV whose key a check's end deletes, so the state holds only the checks that run.
type AsksKV interface {
KV
Delete(ctx context.Context, key string) error
}
// forgetAsks deletes the asks a check put at its end; one that cannot be deleted is said, and the next check of the same
// heads writes the key anew.
func (ch *Checker) forgetAsks(ctx context.Context, key string) {
if err := ch.Asks.Delete(ctx, key); err != nil {
ch.logf("%s: the asks it put could not be forgotten at its end: %v", key, err)
}
}
// recordAsks keeps the asks a check put under its verdict key; nil Asks keeps nothing. A record that cannot be written
// is said: checker_ask_again then refuses the check, as it cannot tell whether its ask waits in a stream.
func (ch *Checker) recordAsks(ctx context.Context, key string, asks []string) {
if ch.Asks == nil {
return
}
data, err := json.Marshal(CheckAsks{Asks: asks})
if err == nil {
_, err = putOnce(ctx, ch.Asks, key, data, func(cur []byte) bool { return string(cur) == string(data) })
}
if err != nil {
ch.logf("%s: the asks it put cannot be kept (%v), so %s refuses it", key, err, AskAgainTool)
}
}
// AskAgainTool asks a candidate's check again by hand.
const AskAgainTool = "checker_ask_again"
// AskedAgainWords open the words of a verdict AskAgainTool wrote; the why follows.
const AskedAgainWords = EndedWords + "asked again by hand (" + AskAgainTool + "): "
// errAskedAgain is the cause a running check is ended with when its verdict was asked again by hand.
var errAskedAgain = errors.New("its verdict was asked again by hand (" + AskAgainTool + ")")
// IsAskedAgain says whether a verdict was written by AskAgainTool.
func IsAskedAgain(v delivery.Verdict) bool {
return IsEnded(v) && strings.HasPrefix(v.Failed.Words, AskedAgainWords)
}
// VerdictEntry is the verdicts state as the ask-again tool writes it: a key's value and revision (nil and 0 for none),
// and a compare-and-set.
type VerdictEntry interface {
Entry(key string) (json.RawMessage, uint64, error)
Update(key string, value []byte, revision uint64) error
}
// AskedAgain is the answer of AskAgainTool.
type AskedAgain struct {
Key string `json:"key"`
Was string `json:"was"`
Said string `json:"said"`
Note string `json:"note"`
}
// AskAgain ends the running verdict of the group's candidate as asked again by hand, saying why. It refuses a group with
// no candidate, a verdict that is not running (a pass or a failure stands until a push or a trunk move), one a step of
// whose pull requests can still run, and one whose asks are not known or one of which no node took yet.
func (r Reader) AskAgain(w VerdictEntry, group, why string) (AskedAgain, error) {
why = strings.TrimSpace(why)
if group == "" || why == "" {
return AskedAgain{}, fmt.Errorf("give the group and why it is asked again")
}
if w == nil {
return AskedAgain{}, fmt.Errorf("the verdicts state cannot be written here")
}
c, ok := r.candidate(group)
if !ok {
return AskedAgain{}, fmt.Errorf("no candidate is held for the group %q", group)
}
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return AskedAgain{}, err
}
raw, rev, err := w.Entry(key)
if err != nil {
return AskedAgain{}, fmt.Errorf("%s cannot be read: %w", key, err)
}
if len(raw) == 0 {
return AskedAgain{}, fmt.Errorf("%s holds no verdict: the checker claims its check when it looks at the candidate (%s)", key, StatusReadTool)
}
v, err := delivery.DecodeVerdict(raw)
if err != nil {
return AskedAgain{}, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
}
if v.State != delivery.VerdictRunning {
return AskedAgain{}, fmt.Errorf("%s is %s, not running: a verdict stands until a push or a trunk move asks it again", key, v.State)
}
steps := r.readSteps()
for _, rs := range steps.running {
if asksOf(c, rs.What) {
return AskedAgain{}, fmt.Errorf("a step of it runs: %s %s on %s since %s (%s); its end judges it", rs.Module, rs.ID, rs.Node, rs.Since, rs.What)
}
}
if r.AskState.Get == nil {
return AskedAgain{}, fmt.Errorf("the checker's asks are not readable here, so whether its ask waits in a build seat's stream cannot be told")
}
kept, err := r.AskState.Get(key)
var put CheckAsks
if err == nil && len(kept) > 0 {
err = json.Unmarshal(kept, &put)
}
if err != nil || len(put.Asks) == 0 {
return AskedAgain{}, fmt.Errorf("the checker names no ask of %s (%v), so whether one waits in a build seat's stream cannot be told; asking again could run it twice", key, err)
}
taken := map[string]bool{}
for _, s := range steps.all {
taken[s.step.ID] = true
}
for _, id := range put.Asks {
if !taken[id] {
return AskedAgain{}, fmt.Errorf("its ask %s waits in its build seat's stream, taken by no node yet: asking again would run it twice; "+
"the seats' backlog is in mesh-delivery-view.build_usage", id)
}
}
words := AskedAgainWords + why
ended := delivery.Verdict{Group: v.Group, Heads: v.Heads, Trunk: v.Trunk, State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Machine: "checker", Words: words}}
data, err := delivery.EncodeVerdict(ended)
if err != nil {
return AskedAgain{}, err
}
if err := w.Update(key, data, rev); err != nil {
return AskedAgain{}, fmt.Errorf("%s changed while it was asked again, so it was not; read it again: %w", key, err)
}
return AskedAgain{Key: key, Was: v.State, Said: words,
Note: "the checker hears it on its watch of its verdicts, ends the wait that judged nothing and asks the check anew, whole; " +
VerdictsReadTool + " shows it running again"}, nil
}
// VerdictWritten takes a change of the checker's verdicts in (its watch, updates only): a verdict AskAgainTool wrote ends
// the check of those heads running in this process, which then considers them again; with none running, the candidate
// that holds them is considered at once. Any other verdict is the checker's own, and left.
func (ch *Checker) VerdictWritten(ctx context.Context, key string, value []byte) {
v, err := delivery.DecodeVerdict(value)
if err != nil || !IsAskedAgain(v) {
return
}
ch.init()
ch.mu.Lock()
if l, ok := ch.live[key]; ok {
l.end(errAskedAgain)
ch.mu.Unlock()
ch.logf("%s: %s; the check running for it is ended and asked again", key, v.Failed.Words)
return
}
var group string
var cand delivery.Candidate
for k, c := range ch.known {
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil && vk == key {
group, cand = k, c
break
}
}
ch.mu.Unlock()
if group == "" {
ch.logf("%s: %s, but no candidate holds these heads, so nothing is asked", key, v.Failed.Words)
return
}
ch.logf("%s: %s; asked again", key, v.Failed.Words)
ch.consider(ctx, group, cand)
}
@@ -115,7 +115,7 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
for _, p := range c.Pulls {
switch head, known := trunk[p.Repository]; {
case !known || head == "":
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: %s", p.Repository, p.Number, UntrunkedWordsOf([]string{p.Repository})))
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
default:
cs.Pulls = append(cs.Pulls, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Trunk: head, Modules: p.Modules, Fps: p.Fps})
cs.Trunk[p.Repository] = head
@@ -133,42 +133,6 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
return cs
}
// UntrunkedWords open the words of a verdict, and of a status, of a pull request that cannot be judged because the gate's
// source.main names no trunk head of its repository (issue 1422): never a failure of the change, and judged whole once the
// trunk head is known. It is said loudly, never left silently "new".
const UntrunkedWords = "cannot be judged: the trunk head of "
// UntrunkedWordsOf are those words for the repositories whose trunk head is not known, in order, each named once.
func UntrunkedWordsOf(repos []string) string {
rs := slices.Clone(repos)
slices.Sort(rs)
rs = slices.Compact(rs)
return UntrunkedWords + strings.Join(rs, ", ") + " is not known: the gate's source.main names none, so there is no commit to merge it onto"
}
// IsUntrunked says whether a verdict says its heads cannot be judged for want of a trunk head (UntrunkedWords).
func IsUntrunked(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && v.Failed.Machine == "checker" && strings.HasPrefix(v.Failed.Words, UntrunkedWords)
}
// leftOf are the pull requests of a candidate whose repository has no trunk head known, with Trunk empty, in repository and
// number order; nil when every repository's is known.
func leftOf(c delivery.Candidate, trunk map[string]string) []Pull {
var out []Pull
for _, p := range c.Pulls {
if trunk[p.Repository] == "" {
out = append(out, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Modules: p.Modules, Fps: p.Fps})
}
}
sort.SliceStable(out, func(i, j int) bool {
if out[i].Repository != out[j].Repository {
return out[i].Repository < out[j].Repository
}
return out[i].Number < out[j].Number
})
return out
}
// GateHeads is the hash the gate matches the members an Approve covers by (mesh-catalog mesh-delivery-gate onestatus.go
// headsOf): delivery.HeadsHash over every pull request of the candidate, hq's included, as
// `<owner/name lowercase>#<number>=<full head sha lowercase>`. A lone pull request is a group of one. Empty for a candidate
@@ -79,8 +79,6 @@ type Checker struct {
}
// Bound is how long one check may run; none when 0 (DefaultBound).
Bound time.Duration
// Asks keeps the asks each running check put, by verdict key (AsksState), for checker_ask_again; nil keeps none.
Asks AsksKV
// Link is the base of the address a status points at; the verdict key follows.
Link string
// Announce says a status was written (EventOwedWritten); nil says nothing.
@@ -250,35 +248,6 @@ func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
ch.consider(ctx, key, c)
}
// VerdictDeleted takes the delete of a verdict key in (the checker's watch of its verdicts, issue 1733): the gate asks the
// operator's Approve only on a verdict, so heads a candidate holds are never left without one. A check of them running in
// this process writes its verdict anew at its end; otherwise the candidate is considered at once and judged again (or said
// again to be unjudgeable). A key no candidate holds is said and left.
func (ch *Checker) VerdictDeleted(ctx context.Context, key string) {
ch.init()
ch.mu.Lock()
if ch.inflight[key] {
ch.mu.Unlock()
ch.logf("%s: deleted while its check runs; its end writes it anew", key)
return
}
var group string
var cand delivery.Candidate
for k, c := range ch.known {
if vk, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil && vk == key {
group, cand = k, c
break
}
}
ch.mu.Unlock()
if group == "" {
ch.logf("%s: deleted, and no candidate holds these heads, so nothing is judged", key)
return
}
ch.logf("%s: deleted; its heads are judged again", key)
ch.consider(ctx, group, cand)
}
// Forget takes a candidate out (its key was deleted): a check of its heads still running is ended, as no candidate holds
// them any more.
func (ch *Checker) Forget(key string) {
@@ -326,17 +295,9 @@ func (ch *Checker) heads() map[string]string {
}
func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) {
heads := ch.heads()
if left := leftOf(c, heads); len(left) > 0 {
// A pull request the checker cannot claim (its repository's trunk head is not known) leaves its whole group
// unjudged, said in a failure verdict (issue 1422, issue 1733): the gate asks the Approve only on a verdict, and a
// pass of the other members would let the group merge with that one never checked.
ch.untrunked(ctx, c, left)
return
}
cs := CaseOf(c, heads)
cs := CaseOf(c, ch.heads())
if cs == nil {
return // no pull request at all: nothing to judge
return // nothing to judge: no trunk known
}
cs.Queued = ch.now()
ch.mu.Lock()
@@ -363,11 +324,7 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
delete(ch.again, vkey)
delete(ch.live, vkey)
ch.mu.Unlock()
again := errors.Is(context.Cause(run), errAskedAgain)
end(nil)
if again && ctx.Err() == nil {
ch.consider(ctx, key, ch.candidate(key, c)) // asked again by hand (AskAgainTool): claimed anew
}
}()
for {
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
@@ -394,76 +351,6 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
}()
}
// untrunked says, loudly, that pull requests of a candidate cannot be judged because the gate's source.main names no trunk
// head of their repository (issue 1422): its verdict key holds a failure that says so, whatever it held before (a running
// verdict left by a check that can no longer run, or a verdict judged on a trunk that is no longer named: issue 1733), which
// checker_verdicts and checker_status show; and every pull request of the candidate gets the status error with
// UntrunkedWords, the members whose trunk is known too, as their group is not judged. It is logged once per head: when a
// status or the verdict is written, never at a look that finds them already said. Once the trunk head is known the verdict
// is stale (it names no trunk) and the heads are judged whole (prepare, claim).
func (ch *Checker) untrunked(ctx context.Context, c delivery.Candidate, left []Pull) {
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
ch.logf("candidates of %s: no verdict key for heads %q, so it cannot be said that they cannot be judged: %v", c.Group, c.HeadsHash, err)
return
}
repos := make([]string, 0, len(left))
for _, p := range left {
repos = append(repos, p.Repository)
}
words := UntrunkedWordsOf(repos)
v := delivery.Verdict{Group: c.Group, Heads: GateHeads(c), Trunk: map[string]string{}, State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Machine: "checker", Words: words}}
data, err := delivery.EncodeVerdict(v)
if err != nil {
ch.logf("%s: %s, and the verdict cannot say so: %v", key, words, err)
return
}
said, err := putOnce(ctx, ch.Verdicts, key, data, func(cur []byte) bool {
held, err := delivery.DecodeVerdict(cur)
return err == nil && IsUntrunked(held) && held.Failed.Words == words // the same words stand; anything else is replaced
})
if err != nil {
ch.logf("%s: %s, and the verdict cannot say so: %v", key, words, err)
}
for _, cp := range c.Pulls {
okey, err := delivery.CheckerOwedKey(cp.Repository, cp.Number, cp.Head.Head)
if err != nil {
ch.logf("%s: %v", key, err)
continue
}
o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "error",
Target: strings.TrimSuffix(ch.Link, "/") + "/" + key, Text: clipLine(words, MaxLine)}
data, err := delivery.EncodeCheckerOwed(o)
if err != nil {
ch.logf("%s: the status of %s cannot be written: %v", key, okey, err)
continue
}
wrote, err := putOnce(ctx, ch.Owed, okey, data, func(cur []byte) bool {
held, err := delivery.DecodeCheckerOwed(cur)
o.V = delivery.Version
return err == nil && delivery.SameOwed(held, o)
})
if err != nil {
ch.logf("%s: the status of %s cannot be written: %v", key, okey, err)
continue
}
if wrote {
said = true
if ch.Announce != nil {
ch.Announce(ctx, EventOwedWritten, okey)
}
}
}
if said {
names := make([]string, 0, len(left))
for _, p := range left {
names = append(names, fmt.Sprintf("%s#%d@%s", p.Repository, p.Number, short12(p.Head)))
}
ch.logf("%s: %s %s; the verdict and every status of its group say so (error), and it is judged once the trunk head is known", key, strings.Join(names, ", "), words)
}
}
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
// the build seats' check stream; a carry, a verdict moved to a newer trunk or sibling commit by its package digests (ADR 0548
// §5), on the recheck stream, pulled after.
@@ -493,8 +380,8 @@ func (ch *Checker) prepare(ctx context.Context, cs *Case) {
return
case IsConflictFailure(v):
return // a head git refused: its next check is whole (ADR 0548 §7)
case IsEnded(v), IsUntrunked(v):
return // ended unjudged when its heads went (issue 1890), or never judged for want of a trunk head (issue 1422): checked whole
case IsEnded(v):
return // ended unjudged when its heads went (issue 1890): held again, they are checked whole
}
cs.Kind = KindTrunk
if v.State == delivery.VerdictPass && ch.stale(v, cs) {
@@ -747,20 +634,10 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
return
}
began := ch.now()
if cs.Watch == nil && ch.Asks != nil {
cs.Watch = func(_ string, step *delivery.Step) {
if step == nil { // put: kept before its wait starts
ch.recordAsks(ctx, cs.Key, cs.Asks())
}
}
defer ch.forgetAsks(ctx, cs.Key)
}
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
out := ch.judge(run, cs)
// An ask the build seat gave up on, or whose step the build module's next process ended because it was replaced under
// it, judged nothing (issue 734, IsLostAsk): the check is asked again under the same claim, its status pending, and after
// LostAskTries it fails as not heard, judged again when the nodes change. A step that ended while its watch was deaf is
// read at the next nudge (HeardEnds) and judged.
// An ask the build seat gave up on judged nothing (issue 734): the check is asked again under the same claim, its status
// pending, and after LostAskTries it fails as not heard, judged again when the nodes change.
for try := 1; IsLostAsk(out.Verdict) && run.Err() == nil; try++ {
if try >= LostAskTries {
out.Verdict.Failed.Words = strings.TrimSpace(out.Verdict.Failed.Words + "\nasked " + fmt.Sprint(LostAskTries) +
@@ -770,8 +647,8 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
ch.logf("%s: %s; asked again (%d of %d)", cs.Key, out.Verdict.Failed.Words, try+1, LostAskTries)
out = ch.judge(run, cs)
}
if ctx.Err() != nil || errors.Is(context.Cause(run), errAskedAgain) {
return // the checker stops, or the verdict was asked again by hand, which wrote it: its next check claims it anew
if ctx.Err() != nil {
return
}
var why superseded
if errors.As(context.Cause(run), &why) {
@@ -812,13 +689,8 @@ func (ch *Checker) try(ctx, run context.Context, key string, cs *Case) {
}
}
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil {
if _, _, found, gerr := ch.Verdicts.Get(ctx, cs.Key); gerr == nil && !found {
_, err = ch.Verdicts.Create(ctx, cs.Key, data) // its key was deleted while it ran (issue 1733): written anew
}
if err != nil {
ch.logf("%s: the verdict could not be written: %v", cs.Key, err)
return
}
ch.logf("%s: the verdict could not be written: %v", cs.Key, err)
return
}
if IsNodesFailure(v) {
ch.mu.Lock()
@@ -239,119 +239,14 @@ func TestAnUnheardForgeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *t
}
}
// Issue 1422: a pull request of a repository the gate's source.main names no trunk head of (one never merged through the
// gate) is never dropped silently. It is not judged, its verdict says it cannot be judged and why, its status is error with
// those words, checker_status names it, and the log says it once per head. Once its trunk head is known it is judged whole;
// in a group, the members whose trunk is known are judged and the others' status says why they are not.
func TestARepositoryWithNoTrunkCannotBeJudgedLoudlyUntilItsTrunkIsKnown(t *testing.T) { // N6/AC6, ADR 0484 clause 11, 1422
func TestARepositoryWithNoTrunkGetsNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clause 11
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
var logged []string
r.ch.Logf = func(f string, a ...any) { logged = append(logged, fmt.Sprintf(f, a...)) }
const unknown = "novox/unknown"
cand := candidateOf(t, "feat/y", pullOf(unknown, 4, head1))
key := keyOf(t, "feat/y", cand)
r.ch.Candidate(ctx, "feat/y", cand)
r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf("novox/unknown", 4, head1)))
r.ch.Wait()
r.ch.Reconsider(ctx) // looked at again: nothing is written or said twice
r.ch.NodesChanged(ctx)
r.ch.Wait()
v := r.verdict(t, key)
if j.count() != 0 || !IsUntrunked(v) || !strings.Contains(v.Failed.Words, "the trunk head of "+unknown+" is not known") {
t.Fatalf("judged %d times, verdict %+v, want none judged and a verdict that says it cannot be judged", j.count(), v)
if j.count() != 0 || len(r.verdicts.vals) != 0 || len(r.owed.vals) != 0 {
t.Errorf("judged %d times, %d verdicts, %d statuses, want none", j.count(), len(r.verdicts.vals), len(r.owed.vals))
}
if s := owedState(t, r, unknown, 4, head1); s != "error" {
t.Errorf("%s#4 owes %q, want error", unknown, s)
}
if n := countWith(logged, "cannot be judged"); n != 1 {
t.Errorf("said %d times in the log, want once for the head: %q", n, logged)
}
reader := func() Reader {
ckey, _ := delivery.CandidatesKey("feat/y")
return Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState(r.verdicts.vals).read(),
Trunks: func() (map[string]string, error) { return r.trunks.Heads(), nil }}
}
st, err := reader().Status()
if err != nil {
t.Fatal(err)
}
if why := st.CannotBeJudged["feat/y"]; !strings.Contains(why, unknown) || len(st.Waiting[KindNew]) != 0 || len(st.Unjudged) != 0 {
t.Errorf("cannot_be_judged %v, waiting %v, unjudged %v: want feat/y named as cannot be judged, not as new", st.CannotBeJudged, st.Waiting, st.Unjudged)
}
if rows, _, err := reader().Verdicts("feat/y"); err != nil || len(rows) != 1 || !strings.Contains(rows[0].Reports[unknown+"#4"], "cannot be judged") {
t.Errorf("checker_verdicts %+v (%v), want the report of %s#4 to say it cannot be judged", rows, err, unknown)
}
// Its trunk head becomes known: the heads are judged whole, the same key rewritten, the status success.
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, unknown: trunkA2})
r.ch.TrunkMoved(ctx)
r.ch.Wait()
if v := r.verdict(t, key); j.count() != 1 || v.State != delivery.VerdictPass || v.Trunk[unknown] != trunkA2 {
t.Errorf("judged %d times, verdict %+v, want one pass on %s", j.count(), v, trunkA2)
}
if s := owedState(t, r, unknown, 4, head1); s != "success" {
t.Errorf("%s#4 owes %q once its trunk is known, want success", unknown, s)
}
// Issue 1733: the gate asks the Approve only on a verdict, so no candidate is left without one. A group with a member
// whose trunk is not known is not judged in part: its verdict fails naming that repository, and every status says so; a
// running verdict left for heads that cannot be judged is replaced; a verdict key deleted is judged again on its delete.
for _, row := range []struct {
name string
pulls []delivery.CandidatePull
running bool // a running verdict is left under the key first
deleted bool // the key is deleted after its verdict
judged int
want string // the verdict's state, and for a failure the repository it names
}{
{name: "a member the checker cannot claim", pulls: []delivery.CandidatePull{pullOf(repoA, 9, head2), pullOf("novox/other", 2, head2)}, want: "novox/other"},
{name: "a running verdict left for heads that cannot be judged", pulls: []delivery.CandidatePull{pullOf(unknown, 5, head2)}, running: true, want: unknown},
{name: "a verdict key deleted", pulls: []delivery.CandidatePull{pullOf(repoA, 5, head2)}, deleted: true, judged: 2, want: delivery.VerdictPass},
} {
t.Run(row.name, func(t *testing.T) {
j := passJudge()
r := newRig(t, j)
cand := candidateOf(t, "feat/z", row.pulls...)
key := keyOf(t, "feat/z", cand)
if row.running {
group, heads, _ := delivery.ParseCheckerVerdictKey(key)
raw, _ := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, State: delivery.VerdictRunning})
_, _ = r.verdicts.Create(ctx, key, raw)
}
r.ch.Candidate(ctx, "feat%2Fz", cand)
r.ch.Wait()
if row.deleted {
_ = r.verdicts.Delete(ctx, key)
r.ch.VerdictDeleted(ctx, key) // the watch of the verdicts hands the delete over
r.ch.Wait()
}
v := r.verdict(t, key)
if row.want == delivery.VerdictPass {
if v.State != delivery.VerdictPass || j.count() != row.judged {
t.Errorf("verdict %s after %d judgings, want a pass after %d", v.State, j.count(), row.judged)
}
return
}
if !IsUntrunked(v) || !strings.Contains(v.Failed.Words, row.want) || j.count() != 0 {
t.Errorf("verdict %+v after %d judgings, want a failure naming %s and none judged", v, j.count(), row.want)
}
for _, p := range row.pulls {
if s := owedState(t, r, p.Repository, p.Number, p.Head.Head); s != "error" {
t.Errorf("%s#%d owes %q, want error", p.Repository, p.Number, s)
}
}
})
}
}
func countWith(lines []string, words string) int {
n := 0
for _, l := range lines {
if strings.Contains(l, words) {
n++
}
}
return n
}
// 1887/A10, ADR 0527 §2: hq is judged as any repository, its checks Go tests, and its pull request gets a verdict and a
@@ -5,7 +5,6 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
@@ -273,36 +272,8 @@ func TestACleanMergeIsFetchedAtItsTrunkAndBothLevelsPass(t *testing.T) { // 1679
t.Errorf("%s was not kept", k)
}
}
// ADR 0567 §12: every check ask carries every node's name for the names check; a check whose names cannot be read, or
// that reads none, is not asked and fails saying so.
for _, c := range []struct {
names []string
err error
fails string
}{{[]string{"anchor", "laptop"}, nil, ""}, {nil, nil, "no node record"}, {nil, errors.New("no bucket"), "no bucket"}} {
b := newBench(t)
b.ex.Nodes = namesOf{c.names, c.err}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
asks := b.asks.Asks()
if c.fails != "" {
if out.Verdict.State != delivery.VerdictFail || len(asks) != 0 || !strings.Contains(out.Verdict.Failed.Words, c.fails) {
t.Errorf("names %v, %v: verdict %+v, %d asks", c.names, c.err, out.Verdict, len(asks))
}
} else if len(asks) == 0 || strings.Join(asks[0].Check.NodeNames, ",") != "anchor,laptop" {
t.Errorf("the check ask carries the names %v", asks)
}
}
}
// namesOf is the nodes' names as a test gives them.
type namesOf struct {
names []string
err error
}
func (n namesOf) Names(context.Context) ([]string, error) { return n.names, n.err }
func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
b := newBench(t)
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/slow": digA, "fixture/other": digC}}
@@ -60,11 +60,3 @@ func (m *memKV) Keys(context.Context) ([]string, error) {
sort.Strings(keys)
return keys, nil
}
func (m *memKV) Delete(_ context.Context, key string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.vals, key)
delete(m.revs, key)
return nil
}
@@ -110,14 +110,8 @@ type Examiner struct {
// waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil
// leaves the composition not run.
Compositions CompositionReader
// Nodes names every node with a record, read from each `<node>.node` of the node-engine's have (novox/hq ADR 0567
// §12): every check ask carries them (CheckJob.NodeNames) for the names check, so no repository lists them. A check
// whose names cannot be read, or that reads none, is not asked and fails saying so; nil hands no names.
Nodes interface {
Names(ctx context.Context) ([]string, error)
}
Now func() time.Time
Logf func(format string, args ...any)
Now func() time.Time
Logf func(format string, args ...any)
}
// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote
@@ -409,16 +403,6 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
p := t.pull
placed := len(needs) > 0
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: levels, Kept: true, Packages: packages}
if st.j.Nodes != nil {
names, err := st.j.Nodes.Names(ctx)
if err == nil && len(names) == 0 {
err = fmt.Errorf("no node record is on the bus")
}
if err != nil {
return fail(t.module, fmt.Sprintf("the check of %s#%d is not asked: the nodes' names the names check needs cannot be read: %v", p.Repository, p.Number, err))
}
job.NodeNames = names
}
if t.runner != "" {
job.Runner, job.RunnerWhen = t.runner, buildGoDir
}
@@ -3,7 +3,6 @@ package check
import (
"strings"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
@@ -20,13 +19,9 @@ import (
// judged again when the nodes change.
const LostAskTries = 3
// IsLostAsk says whether a verdict failed because its ask judged nothing: the build seat gave it up (buildrun.LostWords), or
// the build module that ran its step was replaced under it, and its next process ended the step it left running
// (buildstep.RestartReason; a delivery of the build module, as on 2026-10-28). Neither is the change's failure: the check
// is asked again, as for a given-up ask.
// IsLostAsk says whether a verdict failed because the build seat gave its ask up (buildrun.LostWords).
func IsLostAsk(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && (strings.Contains(v.Failed.Words, buildrun.LostWords) ||
strings.Contains(v.Failed.Words, buildstep.RestartReason))
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, buildrun.LostWords)
}
// IsNotTaken says whether a verdict failed because no build node took its ask within the bound (buildrun.NotTakenWords).
@@ -8,7 +8,6 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
@@ -142,206 +141,30 @@ func TestAnAskGivenUpOrNeverTakenEndsItsTryAsErrorAndItsHeadIsAskedAnew(t *testi
})
}
// troubled is the build seat as a check meets it when its step's end does not reach its wait: the first `restarts` asks'
// steps are ended by the build module's next process (buildstep.RestartReason), and the first `deaf` asks' watches hear
// nothing though their steps end (the lost check of 2026-10-28). Its steps state is read through AskSteps and Steps.
type troubled struct {
*buildrun.FakeAsks
answer func(delivery.StepAsk) delivery.Step
mu sync.Mutex
restarts, deaf int
deafIDs map[string]bool
steps map[string]delivery.Step
}
func (tr *troubled) PublishAsk(ctx context.Context, subject, id string, data []byte) error {
if err := tr.FakeAsks.PublishAsk(ctx, subject, id, data); err != nil {
return err
}
a, err := delivery.DecodeStepAsk(data)
if err != nil {
return err
}
step := tr.answer(a)
step.What = buildstep.AskWhat(a)
tr.mu.Lock()
if tr.restarts > 0 {
tr.restarts--
step.State, step.Reason, step.Result = delivery.StepFailed, buildstep.RestartReason, nil
}
deaf := tr.deaf > 0
if deaf {
tr.deaf--
tr.deafIDs[id] = true
}
tr.steps[id] = step
tr.mu.Unlock()
if !deaf {
tr.Write(step)
}
return nil
}
func (tr *troubled) WatchAsk(ctx context.Context, module, askID string) (<-chan delivery.Step, error) {
tr.mu.Lock()
deaf := tr.deafIDs[askID]
tr.mu.Unlock()
if deaf {
return make(chan delivery.Step), nil // open, and silent for good
}
return tr.FakeAsks.WatchAsk(ctx, module, askID)
}
func (tr *troubled) AskSteps(_ context.Context, _, askID string) ([]delivery.Step, error) {
tr.mu.Lock()
defer tr.mu.Unlock()
if s, ok := tr.steps[askID]; ok {
return []delivery.Step{s}, nil
}
return nil, nil
}
// Steps is the steps state as the read tools read it, with running in place of every step the hand has not let end.
func (tr *troubled) Steps(running bool) StateRead {
tr.mu.Lock()
defer tr.mu.Unlock()
vals := map[string]json.RawMessage{}
for id, s := range tr.steps {
if running {
s.State, s.Ended, s.Started = delivery.StepRunning, time.Time{}, time.Now()
}
raw, _ := delivery.EncodeStep(s)
vals[s.Node+"."+id] = raw
}
return StateRead{Keys: func() ([]string, error) {
var out []string
for k := range vals {
out = append(out, k)
}
return out, nil
}, Get: func(k string) (json.RawMessage, error) { return vals[k], nil }}
}
// kvRead is a bucket in memory as the read tools read it, live.
func kvRead(kv *memKV) StateRead {
return StateRead{Keys: func() ([]string, error) { return kv.Keys(context.Background()) },
Get: func(k string) (json.RawMessage, error) {
v, _, _, err := kv.Get(context.Background(), k)
return v, err
}}
}
// asVerdicts is the rig's verdicts as the ask-again tool reads and writes them.
type asVerdicts struct{ kv *memKV }
func (a asVerdicts) Entry(key string) (json.RawMessage, uint64, error) {
v, rev, _, err := a.kv.Get(context.Background(), key)
return v, rev, err
}
func (a asVerdicts) Update(key string, value []byte, revision uint64) error {
_, err := a.kv.Update(context.Background(), key, value, revision)
return err
}
// Issue 734 and the lost check of 2026-10-28: a check whose ask judged nothing is asked again, and one whose step ended
// unheard is judged. Given up by the build seat once, or its step ended by the build module's next process after a
// delivery replaced it, the check is asked again under its claim and passes; given up every time it fails loudly as not
// heard, judged again when the nodes change. A step whose watch went deaf is read at the next step end of its build module
// (a nudge) and judged with no new ask; one no event reaches is named lost by checker_status, refused by checker_ask_again
// while its step still runs, and asked again by it once its step ended.
func TestACheckWhoseAskJudgedNothingIsAskedAgainOrRead(t *testing.T) {
// Issue 734: a pull request's check whose ask the build seat gave up on is asked again under its claim, its status
// pending, and judged; one given up every time fails as not heard and is judged again when the nodes change.
func TestACheckWhoseAskWasGivenUpIsAskedAgain(t *testing.T) {
ctx := context.Background()
for _, row := range []struct {
name string
lose, restarts, deaf int
how string // "nudge": a step of the build module ends; "hand": checker_ask_again
state string
asks int
lose int
state string
asks int
}{
{name: "given up once", lose: 1, state: delivery.VerdictPass, asks: 2},
{name: "given up every time", lose: LostAskTries, state: delivery.VerdictFail, asks: LostAskTries},
{name: "its build module replaced under it", restarts: 1, state: delivery.VerdictPass, asks: 2},
{name: "replaced under every ask", restarts: LostAskTries, state: delivery.VerdictFail, asks: LostAskTries},
{name: "its watch deaf, a step of its build module ends", deaf: 1, how: "nudge", state: delivery.VerdictPass, asks: 1},
{name: "its watch deaf, asked again by hand", deaf: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
{name: "its watch deaf, asked again by hand after its build module replaced it", deaf: 1, restarts: 1, how: "hand", state: delivery.VerdictPass, asks: 2},
{1, delivery.VerdictPass, 2},
{LostAskTries, delivery.VerdictFail, LostAskTries},
} {
t.Run(row.name, func(t *testing.T) {
b := newBench(t)
var asked func() int
var heard *buildrun.HeardEnds
var tr *troubled
if row.lose > 0 {
l := losing(b, row.lose)
asked = func() int { return len(l.Asks()) }
} else {
tr = &troubled{FakeAsks: &buildrun.FakeAsks{}, answer: b.answer, restarts: row.restarts, deaf: row.deaf,
deafIDs: map[string]bool{}, steps: map[string]delivery.Step{}}
heard = &buildrun.HeardEnds{Inner: tr, Read: tr}
b.ex.Asks = buildrun.Asker{Publisher: tr, Waiter: heard, Bound: 5 * time.Second}
asked = func() int { return len(tr.Asks()) }
}
r := newRig(t, b.ex)
asksKept := newMemKV()
r.ch.Asks = asksKept
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
key := keyOf(t, "feat/x", cand)
r.ch.Candidate(ctx, "feat%2Fx", cand)
if row.deaf > 0 {
for deadline := time.Now().Add(2 * time.Second); heard.Waiting() == 0 && time.Now().Before(deadline); {
time.Sleep(2 * time.Millisecond)
}
if v := r.verdict(t, key); v.State != delivery.VerdictRunning {
t.Fatalf("a check whose step ended unheard = %s, want running until it is read", v.State)
}
}
switch row.how {
case "nudge":
heard.Nudge(buildrun.NodeBuildModule) // another build module's step: not this one's to read
heard.Nudge(buildrun.GoBuildModule)
case "hand":
cands := newMemKV()
ckey, _ := delivery.CandidatesKey("feat/x")
_, _ = cands.Create(ctx, ckey, cand)
none := StateRead{Keys: func() ([]string, error) { return nil, nil }, Get: func(string) (json.RawMessage, error) { return nil, nil }}
rd := Reader{CandidateState: kvRead(cands), VerdictState: kvRead(r.verdicts), AskState: kvRead(asksKept),
StepStates: map[string]StateRead{buildrun.GoBuildModule: none}} // no node took its ask yet
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "waits in its build seat's stream") {
t.Errorf("asked again while its ask waits in the stream (issue 1791's kind): %v", err)
}
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(true)
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf"); err == nil || !strings.Contains(err.Error(), "a step of it runs") {
t.Errorf("asked again while its step runs: %v", err)
}
rd.StepStates[buildrun.GoBuildModule] = tr.Steps(false)
if st, _ := rd.Status(); !strings.Contains(st.Lost["feat/x"], AskAgainTool) {
t.Errorf("status lost = %v, want feat/x named with %s", st.Lost, AskAgainTool)
}
got, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "its watch went deaf")
if err != nil || got.Key != key || !strings.HasPrefix(got.Said, AskedAgainWords) {
t.Fatalf("asked again = %+v, %v", got, err)
}
raw, _, _, _ := r.verdicts.Get(ctx, key)
r.ch.VerdictWritten(ctx, key, raw) // the checker's watch of its verdicts hands it over
}
r.ch.Wait()
v := r.verdict(t, key)
if v.State != row.state || asked() != row.asks {
t.Errorf("verdict %s (%s) after %d asks, want %s after %d", v.State, v.Failed.Words, asked(), row.state, row.asks)
}
if _, _, kept, _ := asksKept.Get(ctx, key); kept {
t.Errorf("the asks of the ended check are still kept under %s", key)
}
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
t.Errorf("a check that judged nothing every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
}
if row.how == "hand" {
rd := Reader{VerdictState: kvRead(r.verdicts), CandidateState: kvRead(newMemKV())}
if _, err := rd.AskAgain(asVerdicts{r.verdicts}, "feat/x", "again"); err == nil {
t.Error("a judged check was asked again")
}
}
})
b := newBench(t)
l := losing(b, row.lose)
r := newRig(t, b.ex)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if v.State != row.state || len(l.Asks()) != row.asks {
t.Errorf("lose %d: verdict %s (%s) after %d asks, want %s after %d", row.lose, v.State, v.Failed.Words, len(l.Asks()), row.state, row.asks)
}
if row.state == delivery.VerdictFail && (!IsLostAsk(v) || !IsNodesFailure(v)) {
t.Errorf("a check given up every time = %q, want lost and judged again when the nodes change", v.Failed.Words)
}
}
}
@@ -47,9 +47,7 @@ type Reader struct {
Runs interface{ CommitOf(module string) string }
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
HearingState StateRead
// AskState is the checker's state asks: the asks each running check put (AsksState), which checker_ask_again reads.
AskState StateRead
Now func() time.Time
Now func() time.Time
}
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
@@ -88,11 +86,6 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
cs := CaseOf(c, v.Trunk)
out.Reports = map[string]string{}
if cs == nil {
if IsUntrunked(v) {
for _, p := range leftOf(c, v.Trunk) {
out.Reports[pullKey(p)] = v.Failed.Words
}
}
return out, nil
}
for _, p := range cs.Pulls {
@@ -308,8 +301,6 @@ type RunningStep struct {
ID string `json:"id"`
Tool string `json:"tool"`
Since string `json:"since,omitempty"`
// What says what the step is (buildstep.AskWhat): an ask's kind, repository@head and levels.
What string `json:"what,omitempty"`
}
// LeftStep is a key a build module left running that no step behind it runs any more, and why.
@@ -332,12 +323,6 @@ type Status struct {
// HeldByTurn are the candidates whose check an approved merge's turn holds off (ADR 0548 §3), by group: the repository and
// the group of the merge that waits. Their check is claimed when that turn goes.
HeldByTurn map[string]string `json:"held_by_turn"`
// CannotBeJudged are the candidates no pull request of which can be judged, by group, with why: the gate's source.main
// names no trunk head of their repositories (issue 1422). They are judged once it does; until then their status is error.
CannotBeJudged map[string]string `json:"cannot_be_judged"`
// Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by
// group, with that step: nothing will judge them (the lost check of 2026-10-28); checker_ask_again asks one again.
Lost map[string]string `json:"lost"`
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
// whose ask is given up waits until its bound.
DeadLetters DeadLettersHeard `json:"dead_letters"`
@@ -401,11 +386,63 @@ func heldBy(t *Turns, c delivery.Candidate, heads map[string]string) string {
// Status reads the build modules' steps and the candidates against their verdicts.
func (r Reader) Status() (Status, error) {
st := Status{RunningByNode: map[string]int{}, Lost: map[string]string{}}
read := r.readSteps()
st.Running, st.LeftRunning, st.Notes = read.running, read.left, read.notes
for _, rs := range st.Running {
st.RunningByNode[rs.Node]++
st := Status{RunningByNode: map[string]int{}}
mods := make([]string, 0, len(r.StepStates))
for m := range r.StepStates {
mods = append(mods, m)
}
sort.Strings(mods)
st.Running, st.LeftRunning = []RunningStep{}, []LeftStep{}
now, bound := time.Now(), r.StepBound
if r.Now != nil {
now = r.Now()
}
if bound <= 0 {
bound = DefaultStepBound
}
for _, m := range mods {
sr := r.StepStates[m]
if sr.Keys == nil || sr.Get == nil {
continue
}
keys, err := sr.Keys()
if err != nil {
st.Notes = append(st.Notes, "the steps of "+m+" cannot be listed: "+err.Error())
continue
}
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
newestFirst(keys)
keys, cut := capKeys(keys, "")
if cut != "" {
st.Notes = append(st.Notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
}
steps := make([]delivery.Step, len(keys))
isRunning := make([]bool, len(keys))
_ = parallel(len(keys), func(i int) error {
raw, err := sr.Get(keys[i])
if err != nil || len(raw) == 0 {
return nil
}
step, err := delivery.DecodeStep(raw)
steps[i], isRunning[i] = step, err == nil && step.State == delivery.StepRunning
return nil
})
for i := range keys {
if !isRunning[i] {
continue
}
step := steps[i]
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool}
if !step.Started.IsZero() {
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
}
if why := r.leftWhy(m, step, now, bound); why != "" {
st.LeftRunning = append(st.LeftRunning, LeftStep{RunningStep: rs, Why: why})
continue
}
st.Running = append(st.Running, rs)
st.RunningByNode[step.Node]++
}
}
if r.HaveState.Get == nil {
st.Notes = append(st.Notes, "the node-engine's have is not readable here, so a step is not checked against its build module being on its node")
@@ -415,7 +452,7 @@ func (r Reader) Status() (Status, error) {
}
st.Unjudged = []string{}
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
st.HeldByTurn, st.CannotBeJudged = map[string]string{}, map[string]string{}
st.HeldByTurn = map[string]string{}
turns := r.turns(&st)
var heads map[string]string
if r.Trunks == nil {
@@ -466,15 +503,6 @@ func (r Reader) Status() (Status, error) {
continue
}
c, raw := row.c, row.raw
if heads != nil && len(leftOf(c, heads)) > 0 {
// Nothing of it can be judged now, whatever its verdict says (issue 1422): never counted as waiting.
var repos []string
for _, p := range leftOf(c, heads) {
repos = append(repos, p.Repository)
}
st.CannotBeJudged[c.Group] = UntrunkedWordsOf(repos)
continue
}
if len(raw) == 0 {
st.Unjudged = append(st.Unjudged, c.Group)
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
@@ -485,16 +513,9 @@ func (r Reader) Status() (Status, error) {
}
v, err := delivery.DecodeVerdict(raw)
switch {
case err != nil:
case v.State == delivery.VerdictRunning:
if why := read.lostWhy(c); why != "" {
st.Lost[c.Group] = why
}
case err != nil || v.State == delivery.VerdictRunning:
case IsNodesFailure(v):
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
case IsUntrunked(v):
// Its trunk head is known now: it waits for its whole check.
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
case heads != nil:
if cs := CaseOf(c, heads); cs != nil && (!sameTrunk(v.Trunk, cs.Trunk) || besideMoved(v.Beside, besideAt(heads, r.Runs))) {
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
@@ -510,118 +531,6 @@ func (r Reader) Status() (Status, error) {
return st, nil
}
// stepsRead is what the build modules' steps states hold: the steps that can still run, those left running, and every
// ask step read, by module, for the candidate it belongs to (stepsOf).
type stepsRead struct {
running []RunningStep
left []LeftStep
all []moduleStep
notes []string
}
type moduleStep struct {
module string
step delivery.Step
}
// readSteps reads the newest MaxReads keys of each build module's steps state.
func (r Reader) readSteps() stepsRead {
out := stepsRead{running: []RunningStep{}, left: []LeftStep{}}
mods := make([]string, 0, len(r.StepStates))
for m := range r.StepStates {
mods = append(mods, m)
}
sort.Strings(mods)
now, bound := time.Now(), r.StepBound
if r.Now != nil {
now = r.Now()
}
if bound <= 0 {
bound = DefaultStepBound
}
for _, m := range mods {
sr := r.StepStates[m]
if sr.Keys == nil || sr.Get == nil {
continue
}
keys, err := sr.Keys()
if err != nil {
out.notes = append(out.notes, "the steps of "+m+" cannot be listed: "+err.Error())
continue
}
// A step that runs started lately: the newest ids are read first, at most MaxReads of them.
newestFirst(keys)
keys, cut := capKeys(keys, "")
if cut != "" {
out.notes = append(out.notes, "the steps of "+m+": read the newest "+fmt.Sprint(MaxReads)+" of its keys; a step older than those is not shown")
}
steps := make([]delivery.Step, len(keys))
decoded := make([]bool, len(keys))
_ = parallel(len(keys), func(i int) error {
raw, err := sr.Get(keys[i])
if err != nil || len(raw) == 0 {
return nil
}
step, err := delivery.DecodeStep(raw)
steps[i], decoded[i] = step, err == nil
return nil
})
for i := range keys {
if !decoded[i] {
continue
}
step := steps[i]
out.all = append(out.all, moduleStep{module: m, step: step})
if step.State != delivery.StepRunning {
continue
}
rs := RunningStep{Module: m, Node: step.Node, ID: step.ID, Tool: step.Tool, What: step.What}
if !step.Started.IsZero() {
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
}
if why := r.leftWhy(m, step, now, bound); why != "" {
out.left = append(out.left, LeftStep{RunningStep: rs, Why: why})
continue
}
out.running = append(out.running, rs)
}
}
return out
}
// asksOf says whether a step is an ask of one of the candidate's pull requests: its What names `<repository>@<head12>`
// (buildstep.AskWhat).
func asksOf(c delivery.Candidate, what string) bool {
for _, p := range c.Pulls {
if p.Repository != "" && strings.Contains(what, p.Repository+"@"+short12(p.Head.Head)) {
return true
}
}
return false
}
// lostWhy says why a candidate whose verdict is running has a check that nothing will judge, or "": no step of its pull
// requests can still run, and its newest one ended.
func (s stepsRead) lostWhy(c delivery.Candidate) string {
for _, rs := range s.running {
if asksOf(c, rs.What) {
return ""
}
}
var last *delivery.Step
for i := range s.all {
st := s.all[i].step
if st.State != delivery.StepRunning && asksOf(c, st.What) && (last == nil || st.Ended.After(last.Ended)) {
last = &s.all[i].step
}
}
if last == nil {
return "" // its ask waits in its build seat's stream: no node took it yet
}
return fmt.Sprintf("its verdict is running, but no step of it runs: its last step %s on %s ended %s at %s and no verdict followed; %s asks it again",
last.ID, last.Node, last.State, last.Ended.UTC().Format(time.RFC3339), AskAgainTool)
}
// DefaultStepBound is how long one check ask's step may run once a build node took it (its key written running; the fetch,
// the digests and the levels), never counting its wait in the stream; past it the check fails naming the node.
const DefaultStepBound = 30 * time.Minute
@@ -36,7 +36,6 @@ func (f fakeState) read() StateRead {
func readerOf(t *testing.T) (Reader, string) {
t.Helper()
r := newRig(t, passJudge())
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA2})
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(hqRepo, 3, head2))
r.ch.Candidate(context.Background(), "feat%2Fx", cand)
r.ch.Wait()
@@ -98,11 +98,11 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
t.Errorf("module = %q", m.Module)
}
// delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets.
if want := []state{{Name: "asks", History: 1}, {Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
if want := []state{{Name: "hearing", History: 1}, {Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains",
"mesh-delivery-assigner.compositions", "mesh-delivery-assigner.placement", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates",
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-gate.turns", "mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
@@ -129,9 +129,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
t.Errorf("streams = %+v, want %+v", m.Streams, want)
}
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check
// is asked again by hand (checker_ask_again).
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them.
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code.
+1 -3
View File
@@ -21,7 +21,6 @@
{ "name": "checks", "events": ["preview.**", "owed.**"] }
],
"state": [
{ "name": "asks", "history": 1 },
{ "name": "hearing", "history": 1 },
{ "name": "kept", "history": 1 },
{ "name": "owed", "history": 5 },
@@ -34,8 +33,8 @@
"mesh-build-go.toolchains",
"mesh-build-node.steps",
"mesh-build-node.toolchains",
"mesh-controller.assignments",
"mesh-delivery-assigner.compositions",
"mesh-delivery-assigner.placement",
"mesh-delivery-builder.builds",
"mesh-delivery-gate.source",
"mesh-delivery-gate.turns",
@@ -50,7 +49,6 @@
"node-build-node"
],
"tools": [
"checker_ask_again",
"checker_kept",
"checker_previews",
"checker_pull",
@@ -91,6 +91,28 @@ func treesVia(ctx context.Context, call caller) bridge.Trees {
}
}
// listedVia asks the forge for the files directly under one directory of a commit through call: gitea_commit_trees with
// listed (issue 1126), answer {listed: {directory: {name: blob id}}}.
func listedVia(ctx context.Context, call caller) bridge.Listed {
return func(owner, repo, sha, dir string) (map[string]string, error) {
raw, err := call(ctx, commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{dir}})
if err != nil {
return nil, err
}
var a struct {
Listed map[string]map[string]string `json:"listed"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return nil, fmt.Errorf("the answer of %s is not {listed}: %w", commitTreesTool, err)
}
if a.Listed == nil {
// A forge module from before listed answers none: placement cannot be read, never read as empty.
return nil, fmt.Errorf("the answer of %s lists no directory; the forge module does not list yet", commitTreesTool)
}
return a.Listed[dir], nil
}
}
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
@@ -72,17 +72,11 @@ accounts {
"mesh.mod.gitea.tool.gitea_open_pulls"
"mesh.mod.gitea.tool.gitea_open_pulls.*"
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_performed.>"
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_inputs.>"
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_performed.>"
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_inputs.>"
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_performed.>"
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_inputs.>"
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_performed"
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_inputs"
"$JS.FC.KV_mesh-delivery-forge-bridge_performed.>"
"$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>"
"$KV.mesh-delivery-forge-bridge_performed.>"
"$KV.mesh-delivery-forge-bridge_inputs.>"
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>"
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>"
"$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>"
@@ -225,7 +219,7 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
}
// Its other state bucket, and the gate's buckets the bridge reads, made by the controller from the gate's manifest
// (mesh-sdk go/delivery MergesBucket, OwedBucket with their histories).
for name, history := range map[string]uint8{performedBucket: 1, inputsBucket: 1, delivery.MergesBucket: delivery.MergesHistory, delivery.OwedBucket: delivery.OwedHistory} {
for name, history := range map[string]uint8{performedBucket: 1, delivery.MergesBucket: delivery.MergesHistory, delivery.OwedBucket: delivery.OwedHistory} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil {
t.Fatal(err)
}
@@ -1,81 +0,0 @@
package main
import (
"context"
"encoding/json"
"log"
"regexp"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-forge-bridge/internal/bridge"
)
// inputsBucket is the module's state bucket inputs (manifest state entry inputs): what the bridge read of a module
// directory, its manifest and lock-file hashes, by the directory's git tree hash, so a merge reads the forge only for the
// directories it changed (issue 1126; internal/bridge kept.go). A tree hash names its bytes, so a value is never stale;
// a merge adds a value only for each directory it changed (a few kilobytes each).
const inputsBucket = delivery.ForgeBridgeModule + "_inputs"
// keptTimeout bounds one read or write of the bucket: the bucket is a saving, never a reason to fail a merge.
const keptTimeout = 2 * time.Second
var treeHash = regexp.MustCompile(`^[0-9a-f]{40}$|^[0-9a-f]{64}$`)
// kept is the bridge's Kept: in memory always, and in the bucket inputs when it is on the bus, so the first merge after a
// start reads only what changed too. A value the bucket cannot give or take is read from the forge, and said.
type kept struct {
kv jetstream.KeyValue // nil when the bucket is not on the bus
logger *log.Logger
mu sync.Mutex
mem map[string]bridge.Inputs
}
func newKept(kv jetstream.KeyValue, logger *log.Logger) *kept {
return &kept{kv: kv, logger: logger, mem: map[string]bridge.Inputs{}}
}
func (k *kept) Get(tree string) (bridge.Inputs, bool) {
k.mu.Lock()
in, ok := k.mem[tree]
k.mu.Unlock()
if ok || k.kv == nil || !treeHash.MatchString(tree) {
return in, ok
}
ctx, cancel := context.WithTimeout(context.Background(), keptTimeout)
defer cancel()
e, err := k.kv.Get(ctx, tree)
if err != nil {
return bridge.Inputs{}, false // not kept (or not readable): read from the forge
}
if err := json.Unmarshal(e.Value(), &in); err != nil || in.Manifest == "" {
k.logger.Printf("%s.%s cannot be read, so its directory is read from the forge: %v", inputsBucket, tree, err)
return bridge.Inputs{}, false
}
k.mu.Lock()
k.mem[tree] = in
k.mu.Unlock()
return in, true
}
func (k *kept) Put(tree string, in bridge.Inputs) {
k.mu.Lock()
k.mem[tree] = in
k.mu.Unlock()
if k.kv == nil || !treeHash.MatchString(tree) {
return
}
data, err := json.Marshal(in)
if err != nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), keptTimeout)
defer cancel()
if _, err := k.kv.Put(ctx, tree, data); err != nil {
k.logger.Printf("%s.%s is not kept, so the next merge reads its directory from the forge again: %v", inputsBucket, tree, err)
}
}
@@ -67,8 +67,8 @@ type publisher interface {
// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
// delivers it again (the server's ack wait is the only clock).
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, logger *log.Logger) {
msg, err := bridge.ConvertKept(m.Subject(), m.Data(), trees, files, kept)
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, list bridge.Listed, logger *log.Logger) {
msg, err := bridge.ConvertWith(m.Subject(), m.Data(), trees, files, list)
switch {
case errors.Is(err, bridge.ErrRetry):
logger.Printf("%v; heard again in %s", err, retryDelay)
@@ -96,9 +96,6 @@ func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees,
logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err)
return
}
if msg.Said != "" {
logger.Printf("%s: %s", m.Subject(), msg.Said)
}
_ = m.Ack()
}
@@ -163,8 +160,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
go onReconnect(ctx, again, said, logger)
// What a merge read of each module directory, by its tree (issue 1126): a merge reads only the directories it changed.
inputs := newKept(process.OptionalBucket(ctx, js, inputsBucket, "a merge's module directories are kept in memory only, so the first merge after a start reads every one", logger.Printf), logger)
cons, err := cred.EventConsumer(ctx, js, eventsStream)
if err != nil {
// Essential: the forge's events are what the bridge carries.
@@ -175,7 +170,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Bounded: the turn is held, and no sync or other event goes on without it.
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
defer cancel()
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, logger)
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), listedVia(hctx, call), logger)
})
}, logger.Printf)
if err != nil {
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388 h1:ycu/sNieTdLpD3bzhZOCXdzcoqXUOxPRmqWV019REpo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261028174713-38dbe1d3a388/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -54,8 +54,6 @@ var LockFiles = []string{"go.sum", "package-lock.json"}
type Message struct {
Event string
Data []byte
// Said is what the bridge did that the log says, beside writing the message; empty for none.
Said string
}
// The subjects of the forge module's events the bridge writes pull messages for (mesh-catalog modules/gitea,
@@ -86,6 +84,20 @@ type pull struct {
PathsTruncated bool `json:"paths_truncated"`
}
// touchesPlacement says whether a pull request may change a placement file: a changed path under
// delivery.PlacementDir, or paths the forge did not say or cut (the safe direction: the files are read).
func (p pull) touchesPlacement() bool {
if p.Paths == nil || p.PathsTruncated {
return true
}
for _, path := range *p.Paths {
if strings.HasPrefix(path, delivery.PlacementDir+"/") {
return true
}
}
return false
}
func unreadable(subject, why string, args ...any) error {
return fmt.Errorf("%w: %s: %s", ErrUnreadable, subject, fmt.Sprintf(why, args...))
}
@@ -95,12 +107,19 @@ func unreadable(subject, why string, args ...any) error {
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
return ConvertKept(subject, data, trees, files, nil)
return ConvertWith(subject, data, trees, files, nil)
}
// ConvertKept is Convert; with kept, a merge reads the forge only for the module directories whose tree hash kept
// does not hold, and keeps what it read (kept.go).
func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kept) (Message, error) {
// Listed asks the forge for the files directly under one directory of a commit, by name to blob id (gitea_commit_trees
// with listed); a directory the commit lacks lists none.
type Listed func(owner, repo, sha, dir string) (map[string]string, error)
// ConvertWith is Convert; with list, a merge of the catalogue repository (delivery.CatalogueRepository) also carries every
// placement file of its merge commit (novox/hq ADR 0549 §2, issue 1126), read whole or not at all: a forge that cannot
// say is ErrRetry, as for the manifests, so the indexer never takes a part of placement for all of it. So does an
// updated pull request of the catalogue that touches placement/ (touchesPlacement), with every placement file at its
// head, which the indexer writes as placement candidates (ADR 0549 §4: the nodes it places answer it before it merges).
func ConvertWith(subject string, data []byte, trees Trees, files Files, list Listed) (Message, error) {
var action string
switch subject {
case UpdatedSubject:
@@ -127,16 +146,7 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
return Message{}, unreadable(subject, "no branch")
}
if subject == MergedSubject && p.Base == Trunk {
return merged(subject, p, trees, files, kept)
}
said := ""
if action == delivery.PullUpdated && p.Base != "" && p.Base != Trunk {
// The checker judges pull requests into main only, and source.main names main's head only (novox/hq ADR 0484 §11): a
// pull request into another base is no candidate, as it is none at the start (OpenPulls) and at its merge. It is
// written closed, so a candidate an older head made is taken out, and said (issue 1422).
action = delivery.PullClosed
said = fmt.Sprintf("%s/%s#%d at %s is into %s, not %s: written closed, so it is no candidate; the checker judges pull requests into %s only (novox/hq ADR 0484 §11)",
p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base, Trunk, Trunk)
return merged(subject, p, trees, files, list)
}
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Action: action}
@@ -148,6 +158,13 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
}
}
if action == delivery.PullUpdated && list != nil && delivery.SameRepository(f.Repository, delivery.CatalogueRepository) && p.touchesPlacement() {
var err error
if f.Placements, err = placementsOf(p.Owner, p.Repo, p.HeadSHA, list, files); err != nil {
return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the head %s: %v", ErrRetry, subject, f.Repository, p.HeadSHA, err)
}
f.PlacementsRead = true
}
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
@@ -156,7 +173,7 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
return Message{Event: event, Data: body, Said: said}, nil
return Message{Event: event, Data: body}, nil
}
// MaxReadsInFlight is how many files of the forge inputsOf asks for at once. A merge into a repository of many
@@ -169,7 +186,6 @@ const MaxReadsInFlight = 8
type read struct {
dir, path string
manifest bool
asked bool
raw []byte
err error
}
@@ -183,14 +199,6 @@ type read struct {
// order the calls finish in, and the error is that of the first failing file in the order of dirs and LockFiles. After
// a failure no further file is asked for, and inputsOf returns only when every call it started has returned.
func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) {
manifests, locks, _, err = inputsRead(owner, repo, ref, dirs, files)
return manifests, locks, err
}
// inputsRead is inputsOf, and also answers, on a failure too, the directories whose manifest and lock files were all
// read, with their manifests and lock hashes: a merge keeps them by their tree, so its next try reads only the rest
// (issue 1126).
func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, whole []string, err error) {
at := func(dir, name string) string {
if dir == "" {
return name
@@ -219,7 +227,6 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
<-slots
break
}
r.asked = true
wg.Add(1)
go func(r *read) {
defer func() { <-slots; wg.Done() }()
@@ -234,16 +241,10 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
wg.Wait()
manifests = map[string]string{}
locks = map[string]map[string]string{}
broken := map[string]bool{} // a directory with a file not read (failed, or never asked after a failure)
for _, r := range reads {
switch {
case r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)):
broken[r.dir] = true
if err == nil {
err = fmt.Errorf("%s: %w", r.path, r.err)
}
case r.err == nil && r.raw == nil && !r.asked:
broken[r.dir] = true
return nil, nil, fmt.Errorf("%s: %w", r.path, r.err)
case r.err != nil: // a lock file the forge does not have
case r.manifest:
manifests[r.dir] = string(r.raw)
@@ -255,29 +256,17 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
locks[r.dir][path.Base(r.path)] = hex.EncodeToString(sum[:])
}
}
for _, dir := range dirs {
if !broken[dir] {
whole = append(whole, dir)
}
}
if err != nil {
for dir := range broken {
delete(manifests, dir)
delete(locks, dir)
}
return manifests, locks, whole, err
}
if len(manifests) == 0 {
manifests = nil
}
if len(locks) == 0 {
locks = nil
}
return manifests, locks, whole, nil
return manifests, locks, nil
}
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Message, error) {
func merged(subject string, p pull, trees Trees, files Files, list Listed) (Message, error) {
if !sha40.MatchString(p.MergeSHA) {
return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
}
@@ -294,20 +283,9 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
names = append(names, dir)
}
slices.Sort(names)
manifests, locks, toRead := keptInputs(names, dirs, kept)
read, readLocks, whole, err := inputsRead(p.Owner, p.Repo, p.MergeSHA, toRead, files)
manifests, locks, err := inputsOf(p.Owner, p.Repo, p.MergeSHA, names, files)
if err != nil {
keep(whole, dirs, read, readLocks, kept) // what was read whole is not asked again at the next try
return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s (%d of %d module directories read from the forge, the others kept by their tree): %v", ErrRetry, subject, repository, p.MergeSHA, len(toRead), len(names), err)
}
keep(toRead, dirs, read, readLocks, kept)
maps.Copy(manifests, read)
maps.Copy(locks, readLocks)
if len(manifests) == 0 {
manifests = nil
}
if len(locks) == 0 {
locks = nil
return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
}
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
if err != nil {
@@ -315,6 +293,12 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
}
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
if list != nil && delivery.SameRepository(repository, delivery.CatalogueRepository) {
if f.Placements, err = placementsOf(p.Owner, p.Repo, p.MergeSHA, list, files); err != nil {
return Message{}, fmt.Errorf("%w: %s: the placement files of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
}
f.PlacementsRead = true
}
body, err := delivery.EncodeForge(f)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
@@ -412,3 +396,30 @@ func OpenMessage(answer []byte) (Message, error) {
}
return Message{Event: delivery.OpenEvent(), Data: body}, nil
}
// placementsOf is every placement file of the catalogue at a commit, by node, its bytes exact: each `<node>.json` directly
// under delivery.PlacementDir. A file whose name is no node is left out and the rest read; the catalogue's check refuses
// such a file before it merges (ADR 0549 §1). Nil when there is none.
func placementsOf(owner, repo, sha string, list Listed, files Files) (map[string]string, error) {
names, err := list(owner, repo, sha, delivery.PlacementDir)
if err != nil {
return nil, err
}
var out map[string]string
for _, name := range slices.Sorted(maps.Keys(names)) {
file := delivery.PlacementDir + "/" + name
node, ok := delivery.PlacementFileNode(file)
if !ok {
continue
}
raw, err := files(owner, repo, file, sha)
if err != nil {
return nil, fmt.Errorf("%s: %w", file, err)
}
if out == nil {
out = map[string]string{}
}
out[node] = string(raw)
}
return out, nil
}
@@ -110,21 +110,6 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
// Issue 1422, ADR 0484 §11: a head pushed to a pull request into another base is written closed, read nothing for, and said;
// one into main is no closed message, and an event naming no base is written as before.
msg, err = Convert(updatedSubject, updated(map[string]any{"base": "nox-mesh"}), nil, nil)
if err != nil {
t.Fatal(err)
}
if f, err = delivery.DecodeForge(msg.Data); err != nil || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
t.Errorf("an update into nox-mesh is %+v (%v), said %q; want it closed and said", f, err, msg.Said)
}
for _, base := range []any{"main", nil} {
msg, err = Convert(updatedSubject, updated(map[string]any{"base": base}), nil, anyFiles)
if f, _ = delivery.DecodeForge(msg.Data); err != nil || f.Action != delivery.PullUpdated || msg.Said != "" {
t.Errorf("an update with base %v is %+v (%v), said %q; want it updated", base, f, err, msg.Said)
}
}
}
func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5
@@ -79,26 +79,47 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
t.Errorf("locks = %v, want %v", f.Locks, want)
}
// ADR 0567: placement is the seat mesh-placement's: a merge of the catalogue carries no placement file and asks the
// forge for none.
forge.calls = nil
msg, err = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get)
if f.Placements != nil || f.PlacementsRead {
t.Errorf("a merge of another repository than the catalogue carries placement: %v", f.Placements)
}
// Issue 1126 (ADR 0549 §2): a merge of the catalogue carries every placement file of its merge commit, by node; a file
// whose name is no node is left out; a listing or a file the forge cannot give is a retry, never a part.
catalogue := mergedBody(map[string]any{"repo": "mesh-catalog"})
forge.have["placement/ace.json"] = []byte(`{"v":1,"node":"ace","modules":[]}`)
forge.have["placement/g14.json"] = []byte(`{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`)
listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3", "a.b.json": "b4"}
var listedAt []string
list := func(owner, repo, sha, dir string) (map[string]string, error) {
listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir)
return listing, nil
}
msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list)
if err != nil {
t.Fatal(err)
}
if f, err = delivery.DecodeForge(msg.Data); err != nil {
t.Fatal(err)
}
if len(f.RetiredPlacements) > 0 || f.RetiredRead || strings.Contains(string(msg.Data), `"placements`) {
t.Errorf("a merge of the catalogue carries placement: %s", msg.Data)
want := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`}
if !f.PlacementsRead || !reflect.DeepEqual(f.Placements, want) || !reflect.DeepEqual(listedAt, []string{"novox/mesh-catalog@" + merge + ":placement"}) {
t.Errorf("placements = %v (read %v), listed at %v", f.Placements, f.PlacementsRead, listedAt)
}
for _, c := range forge.calls {
if strings.HasPrefix(c.path, "placement") {
t.Errorf("the forge was asked for %s", c.path)
}
listing = map[string]string{}
if msg, err = ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); err != nil {
t.Fatal(err)
}
if f, _ = delivery.DecodeForge(msg.Data); !f.PlacementsRead || f.Placements != nil {
t.Errorf("a catalogue with no placement file says it read none: %v, %v", f.Placements, f.PlacementsRead)
}
listing = map[string]string{"gone.json": "b5"}
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") {
t.Errorf("a placement file the forge cannot give: %v", err)
}
failing := func(_, _, _, _ string) (map[string]string, error) { return nil, errors.New("forge down") }
if _, err := ConvertWith(mergedSubject, catalogue, treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get, failing); !errors.Is(err, ErrRetry) {
t.Errorf("a listing the forge cannot give: %v", err)
}
// Issue 1126: a merge reads the forge only for the module directories whose tree changed.
t.Run("a merge reads only the directories whose tree changed", mergeReadsOnlyChangedDirs)
}
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
@@ -125,6 +146,55 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
}
// Issue 1126 (ADR 0549 §4): an updated pull request of the catalogue that touches placement/ carries every placement
// file at its head, read whole or not at all; one that touches none, or of another repository, carries none and lists
// nothing; paths the forge did not say or cut are read as touching; a closed one carries none.
forge = &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest,
"placement/ace.json": []byte(`{"v":1,"node":"ace","modules":[]}`), "placement/g14.json": []byte(`{"v":1,"node":"g14","modules":[]}`)}}
listing := map[string]string{"ace.json": "b1", "g14.json": "b2", "README.md": "b3"}
var listedAt []string
list := func(owner, repo, sha, dir string) (map[string]string, error) {
listedAt = append(listedAt, owner+"/"+repo+"@"+sha+":"+dir)
return listing, nil
}
all := map[string]string{"ace": `{"v":1,"node":"ace","modules":[]}`, "g14": `{"v":1,"node":"g14","modules":[]}`}
catalogue := map[string]any{"repo": "mesh-catalog", "paths": []string{"placement/g14.json"}}
for _, row := range []struct {
name string
subject string
body map[string]any
want map[string]string
read bool
}{
{"a catalogue pull touching placement", updatedSubject, catalogue, all, true},
{"a catalogue pull touching no placement", updatedSubject, map[string]any{"repo": "mesh-catalog"}, nil, false},
{"a catalogue pull whose paths are cut", updatedSubject, map[string]any{"repo": "mesh-catalog", "paths_truncated": true}, all, true},
{"a catalogue pull whose paths are not said", updatedSubject, map[string]any{"repo": "mesh-catalog", "paths": nil}, all, true},
{"another repository's pull touching placement/", updatedSubject, map[string]any{"paths": []string{"placement/g14.json"}}, nil, false},
{"a closed catalogue pull", closedSubject, catalogue, nil, false},
} {
listedAt = nil
body := updated(row.body)
msg, err := ConvertWith(row.subject, body, nil, forge.get, list)
if err != nil {
t.Fatalf("%s: %v", row.name, err)
}
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatalf("%s: %v", row.name, err)
}
if f.PlacementsRead != row.read || !reflect.DeepEqual(f.Placements, row.want) || (len(listedAt) > 0) != row.read {
t.Errorf("%s: placements %v (read %v), listed %v", row.name, f.Placements, f.PlacementsRead, listedAt)
}
if row.read && listedAt[0] != "novox/mesh-catalog@"+head+":placement" {
t.Errorf("%s: listed at %v, want the head", row.name, listedAt)
}
}
listing = map[string]string{"gone.json": "b5"}
if _, err := ConvertWith(updatedSubject, updated(catalogue), nil, forge.get, list); !errors.Is(err, ErrRetry) || !strings.Contains(err.Error(), "placement/gone.json") {
t.Errorf("a placement file the forge cannot give at the head: %v", err)
}
}
func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3
@@ -299,73 +369,3 @@ func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testin
t.Errorf("err = %v, want ErrRetry", err)
}
}
// memKept is a Kept in memory.
type memKept map[string]Inputs
func (k memKept) Get(tree string) (Inputs, bool) { in, ok := k[tree]; return in, ok }
func (k memKept) Put(tree string, in Inputs) { k[tree] = in }
// Issue 1126 (mesh-catalog#535; issue 1458 met the kind first): a merge reads the forge only for the module directories
// whose tree changed. Its message is the same as one read whole; a directory whose tree is kept costs no read.
func mergeReadsOnlyChangedDirs(t *testing.T) {
manifest, sum := real(t, "module.json"), real(t, "go.sum")
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}}
kept := memKept{}
var calls []call
first, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get, kept)
if err != nil {
t.Fatal(err)
}
if len(forge.calls) != 6 {
t.Fatalf("the first merge read %d files, want 6 (a manifest and two lock files for each directory)", len(forge.calls))
}
// The next merge changes modules/b alone: only its three files are read, and the message equals one read whole.
forge.calls = nil
forge.have["modules/b/module.json"] = []byte(`{"module":"b","v":2}`)
trees := map[string]string{"modules/a": "t1", "modules/b": "t3"}
next, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get, kept)
if err != nil {
t.Fatal(err)
}
for _, c := range forge.calls {
if !strings.HasPrefix(c.path, "modules/b/") {
t.Errorf("%s was read although its directory's tree is kept", c.path)
}
}
if len(forge.calls) != 3 {
t.Errorf("the second merge read %d files, want 3", len(forge.calls))
}
whole, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), forge.get)
if err != nil {
t.Fatal(err)
}
if string(next.Data) != string(whole.Data) {
t.Errorf("the message from kept inputs differs from one read whole:\n%s\n%s", next.Data, whole.Data)
}
if string(first.Data) == string(next.Data) {
t.Error("the second merge's message did not change")
}
// A failed read keeps the directories it read whole and nothing of the one that failed, so the next try reads only the
// rest: a merge too large for one try's bound converges over its tries.
kept2 := memKept{}
failing := &forgeFiles{have: forge.have, errs: map[string]error{"modules/b/module.json": errors.New("context deadline exceeded")}}
if _, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2); !errors.Is(err, ErrRetry) {
t.Fatalf("err = %v, want ErrRetry", err)
}
if _, ok := kept2["t1"]; !ok || len(kept2) != 1 { // modules/a's files are asked first, before the failure is seen
t.Errorf("a failed read kept %v, want modules/a's tree t1 alone", kept2)
}
failing.calls = nil
failing.errs = nil
again, err := ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2)
if err != nil {
t.Fatal(err)
}
if string(again.Data) != string(whole.Data) {
t.Errorf("the message of the try after a failure differs from one read whole")
}
if len(failing.calls) != 3 {
t.Errorf("the try after a failure read %d files, want only modules/b's 3", len(failing.calls))
}
}
@@ -1,57 +0,0 @@
package bridge
// A merge of a repository of many modules read three files of the forge for every module, whatever the merge changed:
// the catalogue's 140 modules are 420 reads, which no longer fit the 30 s bound of a handled event while the forge is
// busy after a merge, so the merged message of mesh-catalog#535 was never written and the indexer never saw it (issue
// 1126; issue 1458 met the same kind at 131 modules and raised the reads in flight). A directory's git tree hash names
// every byte under it, so the manifest and the lock-file hashes read once at a tree hash are those of every later commit
// whose directory has the same tree: a merge reads the forge only for the directories it changed.
// Inputs is what the bridge read of one module directory at one tree hash: its manifest and its lock files' hex sha256
// by name (none when it has none).
type Inputs struct {
Manifest string `json:"manifest"`
Locks map[string]string `json:"locks,omitempty"`
}
// Kept holds Inputs by a directory's git tree hash (the bridge's state bucket inputs). Get answers false for a tree it
// does not hold or cannot read; Put is best effort: a value it could not keep is read from the forge next time.
type Kept interface {
Get(tree string) (Inputs, bool)
Put(tree string, in Inputs)
}
// keptInputs splits dirs into the inputs kept for their tree hash and the directories still to read from the forge.
func keptInputs(dirs []string, trees map[string]string, kept Kept) (manifests map[string]string, locks map[string]map[string]string, read []string) {
manifests, locks = map[string]string{}, map[string]map[string]string{}
for _, dir := range dirs {
tree := trees[dir]
if kept == nil || tree == "" {
read = append(read, dir)
continue
}
in, ok := kept.Get(tree)
if !ok {
read = append(read, dir)
continue
}
manifests[dir] = in.Manifest
if len(in.Locks) > 0 {
locks[dir] = in.Locks
}
}
return manifests, locks, read
}
// keep puts what was read of each directory under its tree hash.
func keep(dirs []string, trees map[string]string, manifests map[string]string, locks map[string]map[string]string, kept Kept) {
if kept == nil {
return
}
for _, dir := range dirs {
raw, ok := manifests[dir]
if tree := trees[dir]; ok && tree != "" {
kept.Put(tree, Inputs{Manifest: raw, Locks: locks[dir]})
}
}
}
@@ -103,12 +103,11 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
if m.SecretsOwner != "mesh-delivery-forge-bridge" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// Three buckets: the head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443
// §5), the revision of each key of the gate's buckets the bridge performed, so a key is performed once per
// revision (ADR 0437 §8; issue 1341), and what a merge read of each module directory by its tree, so a merge reads
// the forge only for the directories it changed (issue 1126).
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}}) { // 1279/AC6, 1341, 1126
t.Errorf("state = %+v, want the buckets heads, performed and inputs", m.State)
// Two buckets: the head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443
// §5), and the revision of each key of the gate's buckets the bridge performed, so a key is performed once per
// revision (ADR 0437 §8; issue 1341).
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}}) { // 1279/AC6, 1341
t.Errorf("state = %+v, want the buckets heads and performed", m.State)
}
// No other state, no bus, no seat, no tool, no event (ADR 0437 §4; ADR 0521: no check.status).
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
@@ -12,8 +12,7 @@
],
"state": [
{ "name": "heads", "history": 1 },
{ "name": "performed", "history": 1 },
{ "name": "inputs", "history": 1 }
{ "name": "performed", "history": 1 }
],
"reads": [
"mesh-delivery-gate.merges",
@@ -29,9 +29,7 @@ func Tools(ask asker, now func() time.Time) []stdio.Tool {
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
"writes it to its modules state as a merge would. For a module registered from outside the indexed repositories whose " +
"repository the forge could not read (a 404 before the forge's account could read it). A module the indexer holds " +
"whose record stands behind the head the gate's source names (a merge whose merged message never reached the " +
"indexer) is read again at that head and written when it changed. Answers stored, from which " +
"repository the forge could not read (a 404 before the forge's account could read it). Answers stored, from which " +
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
"indexer_module (mesh-delivery-view) reads the result.",
@@ -49,6 +47,25 @@ func Tools(ask asker, now func() time.Time) []stdio.Tool {
}
return a.Result, nil
}},
{Name: index.PlacementRereadTool,
Description: "Read the catalogue's placement files again now (novox/hq ADR 0559 §8): the indexer asks the forge for the trunk's head and " +
"every open catalogue pull request's, and writes its states placement (placement.<node>) and placement-candidates (<group>.<node>) " +
"as a merge and a pull request's head would, only where a key differs; a candidate key whose file no longer differs, or whose pull " +
"request is closed, is deleted. The recovery for a forge message the indexer missed. Answers the heads read, the keys written or " +
"deleted in each state, or why nothing was (the forge could not say). indexer_placement (mesh-delivery-view) reads the result.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"why": map[string]any{"type": "string", "description": "why it is read again now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := index.NewPlacementReread(str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return a.Reread, nil
}},
}
}
@@ -578,39 +578,23 @@ func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC
}
}
// TestIndexerCountsOnlyThePlacedBuildNodesToolchains (issue 1940, issue 774): the toolchains bucket holds the report of
// n2, where the catalogue no longer places mesh-build-go, at another version than n1's, as ace's v26.12.0 stayed beside
// the v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's
// toolchain alone; when the placement puts mesh-build-go on n2 again, its report written since counts and the builders
// disagree (the guard stays); when it is taken off again, the fingerprint comes back. No restart, no timer: the
// placement's watch (ADR 0549 §3), not the controller's assignments.
func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
// TestIndexerCountsOnlyTheAssignedBuildNodesToolchains (issue 1940): the toolchains bucket holds the report of n2, where
// mesh-build-go was unassigned (its assignment ended) at another version than n1's, as ace's v26.12.0 stayed beside the
// v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's toolchain
// alone; when the controller assigns mesh-build-go to n2 again, its report counts and the builders disagree (the guard
// stays); when the assignment ends again, the fingerprint comes back. No restart, no timer: the assignments' watch.
func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
var placement, toolchains jetstream.KeyValue
place := func(ctx context.Context, t *testing.T, node string, modules ...string) {
p := delivery.Placement{Node: node}
for _, m := range modules {
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
}
key, _ := delivery.PlacementKey(node)
value, err := delivery.EncodePlacement(p)
if err != nil {
t.Fatal(err)
}
if _, err := placement.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
}
var assignments jetstream.KeyValue
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil {
t.Fatal(err)
}
toolchains = kv
for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} {
key, _ := delivery.ToolchainKey(node, "go")
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: version, Recipe: "1", At: time.Now()})
@@ -621,12 +605,18 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
t.Fatal(err)
}
}
// The seat mesh-placement's placement, as the assigner writes it (ADR 0567): n1 places mesh-build-go, n2 nothing.
if placement, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PlacementBucket, History: delivery.PlacementHistory}); err != nil {
// The controller's assignments mirror, in its layout (mesh-controller internal/chain Assignment).
if assignments, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_assignments", History: 1}); err != nil {
t.Fatal(err)
}
place(ctx, t, "n1", "mesh-build-go")
place(ctx, t, "n2")
for key, value := range map[string]string{
"n1.mesh-build-go": `{"node":"n1","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z"}`,
"n2.mesh-build-go": `{"node":"n2","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z","ended":"2026-10-23T08:00:00Z"}`,
} {
if _, err := assignments.Put(ctx, key, []byte(value)); err != nil {
t.Fatal(err)
}
}
})
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
@@ -657,22 +647,16 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
if !agreed(held) {
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", held.FP2, held.None2, held.Toolchains)
}
place(ctx, t, "n2", "mesh-build-go")
key, _ := delivery.ToolchainKey("n2", "go")
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.2", Recipe: "1", At: time.Now().Add(time.Second)})
if err != nil {
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil {
t.Fatal(err)
}
if _, err := toolchains.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
// n2's report, written since its placement, counts: the builders disagree. A fingerprint already computed is kept
// n2's report, written since its assignment, counts: the builders disagree. A fingerprint already computed is kept
// (issue 1940), and only a new one needs the agreement: a merge moving avahi gets none until they agree.
time.Sleep(500 * time.Millisecond)
if e, err := modules.Get(ctx, "avahi"); err != nil {
t.Fatal(err)
} else if m, err := delivery.DecodeModule(e.Value()); err != nil || m.FP2 != held.FP2 {
t.Fatalf("after n2's placement avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
t.Fatalf("after n2's assignment avahi has fp2 %q, none2 %q: want its fingerprint %q kept", m.FP2, m.None2, held.FP2)
}
moved := strings.Replace(string(data), `"newtree"`, `"movedtree"`, 1)
if moved == string(data) {
@@ -685,10 +669,12 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
}
if m := read(disagree); !disagree(m) {
t.Fatalf("within 5 s of a merge after n2's placement avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
t.Fatalf("within 5 s of a merge after n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
}
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil {
t.Fatal(err)
}
place(ctx, t, "n2")
if m := read(agreed); !agreed(m) {
t.Fatalf("within 5 s of n2's placement taken away avahi has fp2 %q, none2 %q", m.FP2, m.None2)
t.Fatalf("within 5 s of n2's unassignment avahi has fp2 %q, none2 %q", m.FP2, m.None2)
}
}
@@ -18,13 +18,11 @@ import (
"os"
"os/signal"
"syscall"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
@@ -147,50 +145,25 @@ type delivered interface {
Ack() error
DoubleAck(ctx context.Context) error
Term() error
Metadata() (*jetstream.MsgMetadata, error)
}
// handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated,
// never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server
// delivers it again (the server's ack wait is the only clock). Each message handled is written to the heard state
// (index.Heard; nil, or a state not on the bus yet, writes none), so indexer_heard reads what the indexer heard of FORGE.
func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Store, logger *log.Logger) {
h := index.Heard{Subject: m.Subject()}
if md, err := m.Metadata(); err == nil && md != nil {
h.Sequence, h.Delivered, h.Published = md.Sequence.Stream, md.NumDelivered, md.Timestamp.UTC()
}
defer func() {
if heard == nil {
return
}
h.Handled = time.Now().UTC()
if err := index.WriteHeard(ctx, heard, h); err != nil && !errors.Is(err, process.ErrNotYet) {
logger.Printf("%s: sequence %d is not written to the heard state: %v", h.Subject, h.Sequence, err)
}
}()
// delivers it again (the server's ack wait is the only clock).
func handle(ctx context.Context, m delivered, ix *index.Indexer, logger *log.Logger) {
f, err := delivery.DecodeForge(m.Data())
if err != nil {
logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err)
h.Outcome, h.Words = index.HeardDropped, err.Error()
_ = m.Term()
return
}
h.Kind, h.Repository, h.Number, h.Branch, h.Commit = f.Kind, f.Repository, f.Number, f.Branch, f.Head
if f.Merge != "" {
h.Commit = f.Merge
}
applied, err := ix.ApplyMessage(ctx, f)
h.Written, h.Late = applied.Written, applied.Late
switch {
switch err := ix.Apply(ctx, f); {
case errors.Is(err, index.ErrUnreadable):
logger.Printf("%s: %v; dropped", m.Subject(), err)
h.Outcome, h.Words = index.HeardDropped, err.Error()
_ = m.Term()
case err != nil:
logger.Printf("%s: %s#%d left for redelivery: %v", m.Subject(), f.Repository, f.Number, err)
h.Outcome, h.Words = index.HeardLeft, err.Error()
default:
h.Outcome = index.HeardApplied
// Waits for the server, so the consumer's info no longer counts the message as unacknowledged when the fill that
// follows asks whether the stream is applied (issue 1280); a failed ack is redelivered after the ack wait, which
// kicks the fill again.
@@ -274,6 +247,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Its taken state (ADR 0556 clause 1) is bound when it appears, as every state a delivery adds (issue 1790): until then
// a merge's taken key is said and not written, and the rest of the indexer runs.
taken := process.AwaitBucket(ctx, js, conn, delivery.TakenBucket, "no repository's taken key is written", logger.Printf)
// Its placement state (ADR 0549 §2) is bound when it appears, as every state a delivery adds (issue 1790): until then a
// catalogue merge's placement is said and not written, and the rest of the indexer runs.
placement := process.AwaitBucket(ctx, js, conn, delivery.PlacementBucket, "no catalogue merge's placement is written", logger.Printf)
// Its placement candidates (issue 1126, ADR 0549 §4) are bound the same way: until the state appears a catalogue pull
// request's placement is said and not written, so no node answers it and its check names every placed node as silent.
placementCandidates := process.AwaitBucket(ctx, js, conn, delivery.PlacementCandidatesBucket, "no catalogue pull request's placement candidate is written", logger.Printf)
// The controller's build records only name the directory of a module whose key names none (fillPaths): without them
// such a module waits for its path, every other is indexed (issue 1945).
builds := process.OptionalBucket(ctx, js, delivery.BuildsBucket, "a module whose key names no directory is not given one from the controller's build records until the indexer starts again", logger.Printf)
@@ -305,7 +284,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
defer calls.Close()
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call)}
Placement: awaitedStore{placement}, PlacementCandidates: awaitedStore{placementCandidates}, ReadPlacement: placementVia(calls.Call), Say: func(event string, body any) error { return calls.Emit(event, body) }}
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
@@ -322,14 +301,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
go func() { sourceErrs <- follow(ctx, sw, each) }()
}
}
// Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the
// placement is settled first, so the toolchains settle through the gate already knowing it.
gate := buildrun.NewToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf)
placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, func(ctx context.Context) {
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
// assignments are settled first, so the toolchains settle through the gate already knowing them.
gate := newToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf)
assignErrs, err := watchAssignments(ctx, js, gate, func(ctx context.Context) {
if err := ix.Recompute(ctx); err != nil {
logger.Printf("writing the modules again after the placement of a build module changed: %v", err)
logger.Printf("writing the modules again after an assignment of a build module changed: %v", err)
}
}, logger.Printf)
}, logger)
if err != nil {
return err
}
@@ -425,13 +404,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Essential: the forge's merges are what the indexer indexes.
return fmt.Errorf("the consumer %s_%s on %s cannot be read (%v): the indexer hears no merge without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", cred.Node, cred.Module, forgeStream, err)
}
// A merge applied while the taken state was missing gets its taken key (issue 1280): when the state appears (Ready runs
// once, now or then), and after each applied message. The gate's source heads never move a key (issue 1126).
// A merge applied while the taken state was missing, and the gate's heads once the forge stream is applied whole, get
// their taken key (issue 1280): when the state appears (Ready runs once, now or then), and after each applied message.
ix.Drained = func(ctx context.Context) (bool, error) {
info, err := cons.Info(ctx)
if err != nil {
return false, err
}
return info.NumPending == 0 && info.NumAckPending == 0, nil
}
taken.Ready(func(jetstream.KeyValue) { ix.KickFill(ctx) })
// What the indexer heard of FORGE (issue 1126 note 76) is bound when its state appears, as every state a delivery adds
// (issue 1790): until then no message is written there, and the rest of the indexer runs.
heard := awaitedStore{process.AwaitBucket(ctx, js, conn, index.HeardBucket, "no FORGE message the indexer handles is written for indexer_heard", logger.Printf)}
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, heard, logger); ix.KickFill(ctx) }, logger.Printf)
cc, err := process.HearEvents(cons, func(m jetstream.Msg) { handle(ctx, m, ix, logger); ix.KickFill(ctx) }, logger.Printf)
if err != nil {
return err
}
@@ -443,7 +426,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
case err := <-sourceErrs:
return err
case err := <-placementErrs:
case err := <-assignErrs:
return err
case err := <-tools.errs:
return err
@@ -3,13 +3,9 @@ package main
import (
"bytes"
"context"
"encoding/json"
"log"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -33,21 +29,6 @@ func (f *fakeMsg) DoubleAck(context.Context) error {
}
func (f *fakeMsg) Term() error { f.termd++; return nil }
// Metadata is the stream's word on the message: its sequence 41, delivered a second time (issue 1126 note 76).
func (f *fakeMsg) Metadata() (*jetstream.MsgMetadata, error) {
return &jetstream.MsgMetadata{Sequence: jetstream.SequencePair{Stream: 41, Consumer: 7}, NumDelivered: 2, Timestamp: time.Date(2026, 10, 28, 13, 7, 42, 0, time.UTC)}, nil
}
// heardOf is the indexer's record of the last FORGE message it handled (index.HeardKey).
func heardOf(t *testing.T, st *memStore) index.Heard {
t.Helper()
var h index.Heard
if err := json.Unmarshal(st.vals[index.HeardKey], &h); err != nil {
t.Fatalf("heard.%s = %q: %v", index.HeardKey, st.vals[index.HeardKey], err)
}
return h
}
type memStore struct{ vals map[string][]byte }
func (m *memStore) Get(_ context.Context, k string) ([]byte, uint64, bool, error) {
@@ -76,32 +57,21 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
data, _ := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
heard := &memStore{vals: map[string][]byte{}}
handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0))
handle(context.Background(), m, ix, log.New(&bytes.Buffer{}, "", 0))
c, err := delivery.DecodeCandidate(st.vals["feat/x"])
if err != nil || len(c.Pulls) != 1 || c.Pulls[0].Modules[0] != "a" || m.acked != 1 || m.doubled != 1 || m.termd != 0 {
t.Errorf("candidate %+v err %v acked %d terminated %d", c, err, m.acked, m.termd)
}
// Issue 1126 note 76: what the indexer heard of FORGE is readable (indexer_heard): subject, sequence, deliveries, time,
// repository, commit and what it did.
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != delivery.ForgePull ||
h.Repository != "novox/mesh-delivery" || h.Number != 12 || h.Commit != head || h.Outcome != index.HeardApplied || h.Handled.IsZero() {
t.Errorf("heard = %+v", h)
}
}
func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 1238 (never redelivered)
var logs bytes.Buffer
st := &memStore{vals: map[string][]byte{}}
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.x.y.pull", data: []byte(`{"v":1,"kind":"pull"}`)}
heard := &memStore{vals: map[string][]byte{}}
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, log.New(&logs, "", 0))
if m.termd != 1 || m.acked != 0 || len(st.vals) != 0 || !strings.Contains(logs.String(), m.subject) {
t.Errorf("terminated %d acked %d keys %d log %q", m.termd, m.acked, len(st.vals), logs.String())
}
if h := heardOf(t, heard); h.Outcome != index.HeardDropped || h.Sequence != 41 || h.Words == "" {
t.Errorf("heard = %+v", h)
}
}
func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value)
@@ -171,35 +171,108 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
}
}
// The forge's tool for the open pull requests, and the trunk branch they are open into.
// The forge's tools the placement reread asks (indexer_placement_reread, novox/hq ADR 0559 §8): a repository's branches
// with their commits, and every open pull request with its branch and head.
const (
openPullsTool = "gitea.gitea_open_pulls"
trunkBranch = "main"
listBranchesTool = "gitea.gitea_list_branches"
openPullsTool = "gitea.gitea_open_pulls"
placementTrunk = "main"
)
// openVia asks the forge which pull requests into the trunk are open now (gitea_open_pulls), every repository's.
func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, error) {
return func(context.Context) ([]delivery.OpenPull, error) {
raw, err := call(openPullsTool, map[string]any{})
// placementVia reads the catalogue's placement now through call: the trunk's head (gitea_list_branches), every placement
// file at it, and each open pull request of the catalogue into the trunk (gitea_open_pulls) with every placement file at
// its head, each read whole or not at all (gitea_commit_trees with listed, gitea_get_file).
func placementVia(call toolCaller) func(ctx context.Context) (index.PlacementRead, error) {
owner, repo, _ := strings.Cut(delivery.CatalogueRepository, "/")
filesAt := func(sha string) (map[string]string, error) {
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha, "listed": []string{delivery.PlacementDir}})
if err != nil {
return nil, err
}
var a struct {
Listed map[string]map[string]string `json:"listed"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.Listed == nil {
return nil, fmt.Errorf("the answer of %s lists no directory: %v", commitTreesTool, err)
}
var out map[string]string
for name := range a.Listed[delivery.PlacementDir] {
file := delivery.PlacementDir + "/" + name
node, ok := delivery.PlacementFileNode(file)
if !ok {
continue
}
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": file, "ref": sha})
if err != nil {
return nil, fmt.Errorf("%s at %s: %w", file, sha, err)
}
var f struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &f); err != nil || f.File == nil || f.File.Content == nil {
return nil, fmt.Errorf("the answer of %s for %s is not {file: {content}}: %v", getFileTool, file, err)
}
if out == nil {
out = map[string]string{}
}
out[node] = *f.File.Content
}
return out, nil
}
return func(context.Context) (index.PlacementRead, error) {
raw, err := call(listBranchesTool, map[string]string{"owner": owner, "repo": repo})
if err != nil {
return index.PlacementRead{}, err
}
var b struct {
Branches []struct {
Name string `json:"name"`
Commit string `json:"commit"`
} `json:"branches"`
}
if err := json.Unmarshal(raw, &b); err != nil {
return index.PlacementRead{}, fmt.Errorf("the answer of %s is not {branches}: %w", listBranchesTool, err)
}
var out index.PlacementRead
for _, br := range b.Branches {
if br.Name == placementTrunk {
out.Trunk = br.Commit
}
}
if out.Trunk == "" {
return index.PlacementRead{}, fmt.Errorf("%s names no branch %s of %s", listBranchesTool, placementTrunk, delivery.CatalogueRepository)
}
if out.Files, err = filesAt(out.Trunk); err != nil {
return index.PlacementRead{}, err
}
raw, err = call(openPullsTool, map[string]any{})
if err != nil {
return index.PlacementRead{}, err
}
var o struct {
Pulls *[]struct {
Owner string `json:"owner"`
Repo string `json:"repo"`
Number int `json:"number"`
Base string `json:"base"`
Owner string `json:"owner"`
Repo string `json:"repo"`
Number int `json:"number"`
Base string `json:"base"`
Head string `json:"head"`
HeadSHA string `json:"head_sha"`
} `json:"pulls"`
}
if err := json.Unmarshal(raw, &o); err != nil || o.Pulls == nil {
return nil, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
return index.PlacementRead{}, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
}
out := []delivery.OpenPull{}
for _, p := range *o.Pulls {
if p.Base == trunkBranch && p.Number > 0 {
out = append(out, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
if !delivery.SameRepository(p.Owner+"/"+p.Repo, delivery.CatalogueRepository) || p.Base != placementTrunk || p.Number <= 0 || p.Head == "" || p.HeadSHA == "" {
continue
}
files, err := filesAt(p.HeadSHA)
if err != nil {
return index.PlacementRead{}, fmt.Errorf("%s#%d: %w", delivery.CatalogueRepository, p.Number, err)
}
out.Pulls = append(out.Pulls, index.PullPlacement{Branch: p.Head, Number: p.Number, Head: p.HeadSHA, Files: files})
}
return out, nil
}
@@ -0,0 +1,211 @@
package main
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
// toolchain agreement this file repairs.
import (
"context"
"encoding/json"
"fmt"
"log"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provider"
)
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
//
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
// key <node>.<module>) hold an open assignment of its build module on its node, and only once it was written since that
// assignment; it is passed on as deleted otherwise, said once, and passed on when its node writes it again. A report
// written before the assignment is the one an earlier run of the module left (issue 1940, 2026-10-25 20:56: mesh-build-node
// assigned to ace again brought ace.typescript v26.12.0 of 2026-10-23 back into the agreement, before the module ran
// there, and the controller's take-in refused every module "builders disagree"). Until the assignments are settled (or where they cannot
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
// refusal is the guard of novox/hq ADR 0465, and it stays.
type toolchainGate struct {
mu sync.Mutex
modules map[string]bool // the build modules whose toolchains pass the gate
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
logf func(format string, a ...any)
known bool // the assignments are settled
standing map[string]time.Time // <node>.<module> with an open assignment -> since when it stands
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
left map[string]string // <module>/<key> -> why it was said left out, so each reason is said once
}
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]time.Time{},
raw: map[string]map[string][]byte{}, left: map[string]string{}}
for _, m := range modules {
g.modules[m] = true
}
return g
}
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
g.mu.Lock()
defer g.mu.Unlock()
if g.raw[module] == nil {
g.raw[module] = map[string][]byte{}
}
if deleted {
delete(g.raw[module], key)
} else {
g.raw[module][key] = value
}
return g.pass(module, key)
}
// Assignment takes one change of the controller's assignments in (at is when the bus wrote that revision) and answers
// whether what it passed on changed.
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool, at time.Time) bool {
node, module, ok := strings.Cut(key, ".")
if !ok || !g.modules[module] {
return false
}
assigned, open := openAssignment(value)
open = open && !deleted
g.mu.Lock()
defer g.mu.Unlock()
was, stood := g.standing[key]
since := assigned
if since.IsZero() {
// A value with no time of its own (an assignment a module pulled in) stands since the revision that opened it.
since = at
if stood {
since = was
}
}
if open == stood && (!open || since.Equal(was)) {
return false
}
if open {
g.standing[key] = since
} else {
delete(g.standing, key)
}
if !g.known {
return false
}
changed := false
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
changed = g.pass(module, k) || changed
}
}
return changed
}
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
// passed on changed.
func (g *toolchainGate) Settled() bool {
g.mu.Lock()
defer g.mu.Unlock()
g.known = true
changed := false
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
changed = g.pass(module, k) || changed
}
}
return changed
}
// pass hands one key on as it stands: its value when its node is assigned and wrote it since (or the assignments are not
// known), deleted otherwise. The caller holds mu.
func (g *toolchainGate) pass(module, key string) bool {
value, held := g.raw[module][key]
if !held {
return g.forward(module, key, nil, true)
}
node, _, err := delivery.ParseToolchainKey(key)
why := ""
if err == nil && g.known {
why = g.leftOut(module, node, value)
}
if why == "" {
delete(g.left, module+"/"+key)
return g.forward(module, key, value, false)
}
if g.left[module+"/"+key] != why {
g.left[module+"/"+key] = why
g.logf("%s.%s is left out of the agreed toolchain: %s (issue 1940)", delivery.ToolchainsBucket(module), key, why)
}
return g.forward(module, key, nil, true)
}
// leftOut is why a node's report is no builder's, or empty when it counts: its node has no open assignment of the module,
// or the report was written before that assignment. A value that is no toolchain is passed on for its reader to refuse.
// The caller holds mu.
func (g *toolchainGate) leftOut(module, node string, value []byte) string {
since, open := g.standing[node+"."+module]
if !open {
return fmt.Sprintf("%s has no assignment on %s, so it is no builder", module, node)
}
tc, err := delivery.DecodeToolchain(value)
if err == nil && tc.At.Before(since) {
return fmt.Sprintf("it was written at %s, before %s's assignment on %s at %s: it counts once %s writes it again",
tc.At.UTC().Format(time.RFC3339), module, node, since.UTC().Format(time.RFC3339), node)
}
return ""
}
// openAssignment is whether a value of the controller's assignments holds an assignment that stands (one that has not
// ended and has no state: a pending one is not on the machine yet; mesh-controller internal/chain Assignment), and when
// it was assigned, zero where the value does not say.
func openAssignment(value []byte) (time.Time, bool) {
var a struct {
Assigned time.Time `json:"assigned"`
Ended time.Time `json:"ended"`
State string `json:"state"`
}
if json.Unmarshal(value, &a) != nil {
return time.Time{}, false
}
return a.Assigned, a.Ended.IsZero() && a.State == ""
}
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
errs := make(chan error, 1)
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
if err != nil {
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) }
w, err := settle(ctx, kv, each)
if err != nil {
// Not fatal either (issue 1945): a watch the bus refuses is the same missing grant as a refused read.
logger.Printf("the bucket %s cannot be watched, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
return errs, nil
}
if gate.Settled() {
changed(ctx)
}
go func() {
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
if gate.Assignment(e.Key(), e.Value(), !opOf(e), e.Created()) {
changed(ctx)
}
})
}()
return errs, nil
}
@@ -0,0 +1,165 @@
package main
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
import (
"fmt"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
func gateReport(t *testing.T, node, version string) (string, []byte) {
t.Helper()
key, err := delivery.ToolchainKey(node, "typescript")
if err != nil {
t.Fatal(err)
}
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
if err != nil {
t.Fatal(err)
}
return key, value
}
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
return tc.Version, none
}
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
key, value := gateReport(t, node, version)
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
}
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
}
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
if !gate.Settled() {
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
}
if len(said) != 1 {
t.Fatalf("ace's report is said left out once; said %v", said)
}
// ace reports again while unassigned (an old value seen again): still left out.
key, value := gateReport(t, "ace", "v26.12.0")
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("an unassigned node's report changed what was passed on")
}
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
t.Fatal("assigning ace again passed nothing on")
}
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("builders that run different versions still disagree; got %q", none)
}
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
t.Fatal("a pending assignment is no assignment on the machine")
}
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
t.Fatalf("got %q, %q", v, none)
}
gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
if v, _ := agreedOf(set); v != "v26.12.1" {
t.Fatalf("got %q", v)
}
// A report deleted from the bucket goes whether its node is assigned or not.
key, _ = gateReport(t, "g14", "v26.12.1")
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
t.Fatal("deleting g14's report passed nothing on")
}
if n := len(set.Entries("typescript")); n != 2 {
t.Fatalf("novox and shanks are left; got %d entries", n)
}
}
// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
key, _ := gateReport(t, node, version)
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
if err != nil {
t.Fatal(err)
}
return key, value
}
for _, c := range []struct {
name string
assignment string // ace's assignment value
revision time.Time // when the bus wrote it
at time.Time // when ace wrote its report
counts bool
}{
{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
} {
t.Run(c.name, func(t *testing.T) {
set := &buildrun.ToolchainSet{}
var said []string
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
for _, node := range []string{"g14", "novox", "shanks"} {
gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
key, value := gateReport(t, node, "v26.12.1")
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
}
gate.Settled()
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
t.Fatal("an assignment with no report passed something on")
}
key, value := report(t, "ace", "v26.12.0", c.at)
if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
}
// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
t.Fatal("the same assignment written again changed what was passed on")
}
v, none := agreedOf(set)
if c.counts {
if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
}
return
}
if v != "v26.12.1" || none != "" {
t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
}
if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
t.Fatalf("said %q, want once why ace's report is left out", said)
}
// ace writes its report again since it was assigned: it counts.
key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
t.Fatal("ace's new report was not passed on")
}
if n := len(set.Entries("typescript")); n != 4 {
t.Fatalf("four builders report; got %d", n)
}
})
}
}
@@ -14,6 +14,7 @@ import (
// takerIn is the part of the indexer a takeins value asks through.
type takerIn interface {
TakeIn(ctx context.Context, module string) index.TakenIn
RereadPlacement(ctx context.Context) index.PlacementReread
}
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
@@ -31,9 +32,15 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
if !a.Answered.IsZero() {
return
}
r := ix.TakeIn(ctx, a.Module)
a.Result, a.Answered = &r, time.Now().UTC()
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
if a.Placement {
r := ix.RereadPlacement(ctx)
a.Reread, a.Answered = &r, time.Now().UTC()
logf("a placement reread asked by %s (%s): %s", a.By, a.Why, r.Answer)
} else {
r := ix.TakeIn(ctx, a.Module)
a.Result, a.Answered = &r, time.Now().UTC()
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
}
data, err := json.Marshal(a)
if err != nil {
return
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952 h1:TmveT3Bija643RGWkilvkzP/h8UWlVvHFZZtQfJK5oU=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027193820-8ad9372b1952/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -65,10 +65,19 @@ type Indexer struct {
// Contains says whether a commit is the second or an ancestor of it (gitea.gitea_contains).
Contains func(ctx context.Context, repository, commit, in string) (bool, error)
Logf func(format string, args ...any)
// ReadOpen asks the forge which pull requests into the trunk are open now (gitea_open_pulls). A catalogue pull message
// that finds the candidates of another pull request asks it and deletes those of a pull request no longer open, so a
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
ReadOpen func(ctx context.Context) ([]delivery.OpenPull, error)
// Placement is the placement bucket (delivery.PlacementBucket, ADR 0549 §2); nil writes no placement.
Placement Store
// PlacementCandidates is the placement candidates bucket (delivery.PlacementCandidatesBucket, issue 1126); nil writes
// no candidate (placement_candidates.go).
PlacementCandidates Store
// ReadPlacement asks the forge for the catalogue's placement now, at the trunk's head and each open pull request's
// (indexer_placement_reread, ADR 0559 §8); nil rereads none.
ReadPlacement func(ctx context.Context) (PlacementRead, error)
// Say says one of the indexer's events (placement-changed); nil says none.
Say func(event string, body any) error
// Drained says whether the indexer has applied every message of the forge stream (the consumer has none pending and none
// unacknowledged); FillTaken writes the source's heads only then. Nil fills none from the source (taken.go).
Drained func(ctx context.Context) (bool, error)
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
@@ -96,54 +105,47 @@ func (ix *Indexer) logf(format string, args ...any) {
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
// merged message also writes the fingerprint of each module of its repository (modules.go).
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
_, err := ix.ApplyMessage(ctx, f)
return err
}
// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each
// applied merged message is said in one line: its repository, commit and pull request, and the modules written.
func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, error) {
var applied Applied
if f.Kind == delivery.ForgeOpen {
return applied, ix.applyOpen(ctx, f)
return ix.applyOpen(ctx, f)
}
if f.Kind != delivery.ForgePull && f.Kind != delivery.ForgeMerged {
return applied, nil
return nil
}
if f.Kind == delivery.ForgeMerged && ix.Modules != nil {
w, err := ix.applyMerged(ctx, f)
applied = Applied{Written: w.written, Late: w.late}
if err != nil {
return applied, err
if err := ix.applyMerged(ctx, f); err != nil {
return err
}
ix.logf("%s", w.said(f.Number))
if w.late == "" { // a late message leaves the taken key, which names a later merge
if err := ix.writeTakenKnown(ctx, f, w.asked); err != nil {
return applied, err
}
if err := ix.writeTaken(ctx, f); err != nil {
return err
}
ix.kickRetry(ctx) // a registered module the forge could not read is read again on a merge, in the background (issue 1945)
}
if err := ix.applyPlacement(ctx, f); err != nil {
return err
}
if err := ix.applyPlacementCandidates(ctx, f); err != nil {
return err
}
var digests map[string]string
var held map[string]delivery.Module
if f.Kind == delivery.ForgePull {
var err error
if digests, err = ix.keepManifests(ctx, f); err != nil {
return applied, err
return err
}
if held, err = ix.heldModules(ctx); err != nil {
return applied, err
return err
}
}
if trunkLike[f.Branch] {
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
return applied, nil
return nil
}
key, err := delivery.CandidatesKey(f.Branch)
if err != nil {
return applied, fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
}
return applied, write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
var cur *delivery.Candidate
if found {
c, err := delivery.DecodeCandidate(value)
@@ -178,6 +180,14 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
for _, p := range f.Open {
named[p] = true
}
if ix.PlacementCandidates != nil {
// A pull request the forge no longer has open keeps no placement candidate (issue 1126).
if err := ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
return !named[delivery.OpenPull{Repository: c.Pull.Repository, Number: c.Pull.Number}]
}, "its pull request is not open"); err != nil {
return err
}
}
keys, err := ix.Store.Keys(ctx)
if err != nil {
return fmt.Errorf("listing the candidates: %w", err)
@@ -1,60 +0,0 @@
package index
import (
"context"
"encoding/json"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// What the indexer heard of the forge bridge's stream FORGE (issue 1126 note 76): no tool read FORGE's messages, so a merge
// written back over a later one could not be traced to a late or repeated message. The indexer writes, for each message
// it handles, one value under the key forge of its state heard: the message's subject, stream sequence, how often the
// server delivered it, when it was published and handled, what it names and what the indexer did with it. The state keeps
// the last 64 values (its history); mesh-delivery-view's indexer_heard reads them, newest first, with their times.
const (
HeardState = "heard"
HeardBucket = delivery.IndexerModule + "_" + HeardState
HeardKey = "forge"
// What the indexer did with a message.
HeardApplied = "applied"
HeardLeft = "left for redelivery"
HeardDropped = "dropped"
)
// Heard is one FORGE message as the indexer handled it.
type Heard struct {
V int `json:"v"`
Subject string `json:"subject"`
Sequence uint64 `json:"sequence"` // the stream's sequence of the message
Delivered uint64 `json:"delivered,omitempty"` // how often the server delivered it to the indexer, this time included
Published time.Time `json:"published,omitzero"`
Handled time.Time `json:"handled"`
Kind string `json:"kind,omitempty"`
Repository string `json:"repository,omitempty"`
Number int `json:"number,omitempty"`
Branch string `json:"branch,omitempty"`
Commit string `json:"commit,omitempty"` // the merge commit of a merged message, else the pull request's head
Outcome string `json:"outcome"`
Words string `json:"words,omitempty"` // why it was left or dropped
Written []string `json:"written,omitempty"` // the modules a merged message wrote
Late string `json:"late,omitempty"` // the taken merge a merged message is older than: nothing of it was written (order.go)
}
// Applied is what Apply did with a merged message's modules: the keys written, and, for a message older than the
// repository's taken merge, that merge (nothing was written).
type Applied struct {
Written []string
Late string
}
// WriteHeard writes h as the newest value of heard.forge.
func WriteHeard(ctx context.Context, st Store, h Heard) error {
h.V = 1
data, err := json.Marshal(h)
if err != nil {
return err
}
return write(ctx, st, HeardKey, func([]byte, bool) ([]byte, bool, error) { return data, false, nil })
}
@@ -43,6 +43,8 @@ func withManifests(f delivery.Forge, manifests map[string]string, locks map[stri
}
func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1
// A merge's placement files are written as its manifests are (issue 1126): folded here to hold the count.
placementIsWrittenAtAMergeOnlyWhereItChanged(t)
avahi := catalogueManifest(t, "avahi")
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
@@ -24,30 +24,21 @@ import (
// clause 7). Two directories naming one module write the one the bucket holds, or the first by name, and say the other.
// A value the module already holds but for its commit is not written again: its commit stays the one its fingerprint first
// appeared at (writeOverlay), so a merge writes only the modules it changes.
//
// A merge never moves a module backwards (order.go): a merged message older than the repository's taken merge writes and
// deletes nothing, and is said; it is asked once, before anything is written.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWrite, error) {
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
return ix.applyManifests(ctx, f, true)
}
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
// drops none.
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) {
w := newMergeWrite(f)
if drop {
if err := ix.lateMerge(ctx, w); err != nil || w.late != "" {
return w, err
}
}
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) error {
digests, err := ix.keepManifests(ctx, f)
if err != nil {
return w, err
return err
}
held, err := ix.heldModules(ctx)
if err != nil {
return w, err
return err
}
objects, namedErr := ix.namedObjects(ctx, f)
overlay := map[string]delivery.Module{}
@@ -102,11 +93,11 @@ func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bo
// (issue 1900). A message that carries no manifests (an older writer) or one that cannot be read whole says nothing.
if drop && len(f.Manifests) > 0 && every {
if err := ix.dropRemoved(ctx, f.Repository, held, func(key string, _ delivery.Module) bool { return named[key] }); err != nil {
return w, err
return err
}
}
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
return w, ix.recomputeWith(ctx, overlay, w)
return ix.recomputeWith(ctx, overlay)
}
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
@@ -3,8 +3,6 @@ package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"testing"
@@ -99,17 +97,10 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
commit map[string]string
// taken is the merge the repository's taken key names after then (issue 1280, ADR 0556 clause 1).
taken string
// err says then fails (the message is left for redelivery); said is a line the indexer's log (or that error) must hold.
err bool
said string
// contains, when set, is the forge's order, and asks how often then asks it (one question per merged message).
contains func(context.Context, string, string, string) (bool, error)
asks int
}{
{name: "the same merge again", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
then: func(ix *Indexer) error { return ix.Apply(context.Background(), mergedMsg(repo, three)) }},
{name: "a merge touching one of three modules", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
said: "novox/mesh-delivery at cccccccccccc (#12): merged message applied; modules written: a",
then: func(ix *Indexer) error {
m := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
m.Merge = next
@@ -148,30 +139,32 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
{name: "a source head whose merge was not applied is not a taken key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: "",
then: func(ix *Indexer) error { // the forge stream still holds the merge: the consumer is not drained
delete(ix.Taken.(*memStore).vals, takenKey(repo))
ix.Drained = func(context.Context) (bool, error) { return false, nil }
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
return err
}
return ix.FillTaken(context.Background())
}},
// Issue 1126 (mesh-catalog#535): the forge bridge never wrote the merged message of 9fd9569, so the forge stream was
// drained with the merge not applied; the source head moved the key all the same, and the view said the merge was
// on the machines. A key moves only by a merge applied (ADR 0556 clause 1), never by the source's word.
{name: "a source head of a drained forge stream whose merge was never heard leaves the key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
then: func(ix *Indexer) error {
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == next, nil }
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
return err
}
return ix.FillTaken(context.Background())
}},
{name: "a source head of a drained forge stream fills no missing key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: "",
{name: "a source head of a drained forge stream fills a missing key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
then: func(ix *Indexer) error {
delete(ix.Taken.(*memStore).vals, takenKey(repo))
ix.Drained = func(context.Context) (bool, error) { return true, nil }
if err := ix.ApplySource(context.Background(), 8, sourceOf(repo, next)); err != nil {
return err
}
return ix.FillTaken(context.Background())
}},
{name: "filling from an older source head leaves a later key", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: next,
then: func(ix *Indexer) error {
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == merge && in == next, nil }
m := mergedMsg(repo, three)
m.Merge = next
if err := ix.Apply(context.Background(), m); err != nil {
return err
}
ix.Drained = func(context.Context) (bool, error) { return true, nil }
return ix.FillTaken(context.Background()) // the source still names merge
}},
{name: "an earlier merge heard again after a later one leaves taken", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
then: func(ix *Indexer) error {
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) { return commit == next && in == merge, nil }
@@ -179,78 +172,6 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
m.Merge = next
return ix.Apply(context.Background(), m)
}},
// Issue 1126 note 76: mesh-catalog's merge 9cdac09a was written back over the later 69056948, rolling kernel and
// mesh-build-image back on the nodes. A merge never moves a module backwards: the held commit must be in the incoming one.
{name: "an earlier merge heard again after a later one moves no module back", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
said: "the merged message is older than taken.novox/mesh-delivery at cccccccccccc",
then: func(ix *Indexer) error {
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
return commit == in || commit == merge && in == next, nil
}
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
later.Merge = next
if err := ix.Apply(context.Background(), later); err != nil {
return err
}
return ix.Apply(context.Background(), mergedMsg(repo, three))
}},
{name: "an earlier merge that lacks a later merge's module deletes nothing", writes: 1, commit: map[string]string{"a": merge, "d": next}, taken: next,
said: "the merged message is older than taken.novox/mesh-delivery at cccccccccccc",
then: func(ix *Indexer) error {
ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
return commit == in || commit == merge && in == next, nil
}
later := mergedMsg(repo, map[string]string{"modules/a": "treeA", "modules/b": "treeB", "modules/c": "treeC", "modules/d": "treeD"})
later.Merge = next
if err := ix.Apply(context.Background(), later); err != nil {
return err
}
return ix.Apply(context.Background(), mergedMsg(repo, three))
}},
// Review round 2: a 502 whose words carry "404" inside a sha is no not-found; the late message is left for redelivery.
{name: "a late message whose order the forge fails with a 502 writes nothing and is left for redelivery", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next, err: true,
said: "502 Bad Gateway",
then: func(ix *Indexer) error {
ix.Contains = func(context.Context, string, string, string) (bool, error) { return false, nil }
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
later.Merge = next
if err := ix.Apply(context.Background(), later); err != nil {
return err
}
ix.Contains = func(context.Context, string, string, string) (bool, error) {
return false, errors.New("Gitea API /repos/x/y/compare/aaa404aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa...bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb: 502 Bad Gateway")
}
return ix.Apply(context.Background(), mergedMsg(repo, three))
}},
{name: "a forge that cannot say the order writes nothing and leaves the message for redelivery", writes: 0, commit: map[string]string{"a": merge}, taken: merge, err: true,
said: "cannot be read",
then: func(ix *Indexer) error {
ix.Contains = func(context.Context, string, string, string) (bool, error) {
return false, errors.New("the forge did not answer")
}
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
later.Merge = next
return ix.Apply(context.Background(), later)
}},
// Review of 77d738d: only a merge older than the one taken in is kept back. A diverged or rewritten history, or a
// taken merge the forge no longer has, is not older, and one question per message is asked.
{name: "a merge of a diverged history is written, asking the forge once", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
contains: func(context.Context, string, string, string) (bool, error) { return false, nil }, asks: 1,
then: func(ix *Indexer) error {
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
later.Merge = next
return ix.Apply(context.Background(), later)
}},
{name: "a taken merge the forge no longer has stops no later merge", writes: 1, commit: map[string]string{"a": next, "b": merge, "c": merge}, taken: next,
said: "is not known to be older",
then: func(ix *Indexer) error {
ix.Contains = func(context.Context, string, string, string) (bool, error) {
return false, errors.New("gitea_contains: the forge answered: 404 Not Found")
}
later := mergedMsg(repo, map[string]string{"modules/a": "treeA2", "modules/b": "treeB", "modules/c": "treeC"})
later.Merge = next
return ix.Apply(context.Background(), later)
}},
{name: "a later source revision of another repository's merge", writes: 0, commit: map[string]string{"a": merge, "b": merge, "c": merge}, taken: merge,
then: func(ix *Indexer) error {
return ix.ApplySource(context.Background(), 9, delivery.Source{Group: "g", Heads: map[string]string{repo: merge, "novox/mesh-catalog": otherMerge}})
@@ -268,27 +189,8 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
t.Fatal(err)
}
writes := mods.writes
asks := 0
if tc.contains != nil {
ix.Contains = func(ctx context.Context, repo, commit, in string) (bool, error) {
asks++
return tc.contains(ctx, repo, commit, in)
}
}
var said strings.Builder
ix.Logf = func(format string, args ...any) { fmt.Fprintf(&said, format+"\n", args...) }
err := tc.then(ix)
if tc.contains != nil && asks != tc.asks {
t.Errorf("the forge was asked %d times, want %d", asks, tc.asks)
}
if (err != nil) != tc.err {
t.Fatalf("then = %v, want an error %v", err, tc.err)
}
if err != nil {
said.WriteString(err.Error()) // what handle logs as left for redelivery
}
if !strings.Contains(said.String(), tc.said) {
t.Errorf("the log %q does not say %q", said.String(), tc.said)
if err := tc.then(ix); err != nil {
t.Fatal(err)
}
if got := mods.writes - writes; got != tc.writes {
t.Errorf("%d values written, want %d", got, tc.writes)
@@ -1,118 +0,0 @@
package index
import (
"context"
"errors"
"fmt"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// A merge never moves a module backwards (issue 1126, note 76). The indexer wrote mesh-catalog's merge 9cdac09a back over
// the later 69056948 for kernel and mesh-build-image: applyManifests and writeOverlay wrote whatever commit a merged
// message carried, so a late or repeated message rolled the modules, and the nodes that want them, back.
//
// The question is whether the incoming commit is OLDER: the forge says the incoming commit is in the later one and the
// two differ (Contains(incoming, in: later)). Only then is it kept back. A diverged or rewritten history answers no, and
// is written forward; a commit the forge no longer has (a 404) is not known to be older, is written, and said. A forge
// that cannot say for another reason (a timeout) writes nothing and is an error: the server delivers the message again.
//
// - A merged message asks once, before it writes anything: is its merge older than the repository's taken merge (the
// last merge applied whole, taken.go)? Then the whole message is late: no module is written or deleted, the taken key
// stays, and the log says so.
// - A take-in of a held module asks once for that module: is the source's head older than the module's record
// (takeInHeld)?
// mergeWrite is one merge's write of the modules: its repository and commit, whether it is late (older than the taken
// merge), the taken merge it was compared with and the answer, and the module keys written, for the line the indexer says
// per applied merged message.
type mergeWrite struct {
repository, merge string
late string // the taken merge this one is older than, or ""
asked string // the taken merge the order was asked against, or "" when none was asked
written []string
}
func newMergeWrite(f delivery.Forge) *mergeWrite {
return &mergeWrite{repository: f.Repository, merge: f.Merge}
}
// notFound says the forge answered that a commit it was asked about does not exist: gitea's status ": 404 ", matched as
// readVia matches it (cmd/mesh-delivery-indexer named.go). A bare "404" is not enough: the error's words carry both shas,
// and a 502 naming a sha with 404 in it would read as not older and write a late message backwards. The thorough form,
// a typed not-found answered by gitea_contains, is noted on issue 1126.
func notFound(err error) bool { return err != nil && strings.Contains(err.Error(), ": 404 ") }
// older says commit is older than later: the forge says commit is in later and the two differ. A commit the forge does not
// have is not known to be older (false, said); a forge that cannot say otherwise is an error. Without a forge to ask
// (Contains nil) nothing is older.
func (ix *Indexer) older(ctx context.Context, repository, commit, later string) (bool, error) {
if ix.Contains == nil || commit == "" || later == "" || commit == later {
return false, nil
}
in, err := ix.Contains(ctx, repository, commit, later)
if notFound(err) {
ix.logf("%s: the forge does not have %.12s or %.12s (a rewritten history?), so %.12s is not known to be older and is written: %v", repository, commit, later, commit, err)
return false, nil
}
if err != nil {
return false, fmt.Errorf("%s: whether %.12s is older than %.12s cannot be read, so nothing of it is written (a merge never moves a module backwards, issue 1126): %w", repository, commit, later, err)
}
return in, nil
}
// lateMerge asks, before a merged message writes anything, whether its merge is older than the repository's taken merge
// (one forge call). A taken state not on the bus yet, or no taken key, asks nothing.
func (ix *Indexer) lateMerge(ctx context.Context, w *mergeWrite) error {
if ix.Taken == nil || w.merge == "" {
return nil
}
key, err := delivery.RepositoryKey(w.repository)
if err != nil {
return nil // writeTaken says it
}
value, _, found, err := ix.Taken.Get(ctx, key)
if errors.Is(err, process.ErrNotYet) || !found {
return nil
}
if err != nil {
return fmt.Errorf("taken.%s: %w", key, err)
}
cur, err := delivery.DecodeTaken(value)
if err != nil || cur.Merge == "" || cur.Merge == w.merge {
return nil
}
old, err := ix.older(ctx, w.repository, w.merge, cur.Merge)
if err != nil {
return err
}
w.asked = cur.Merge
if old {
w.late = cur.Merge
ix.logf("%s at %.12s: the merged message is older than taken.%s at %.12s, so no module is written or deleted (a late or repeated message; a merge never moves a module backwards, issue 1126)",
w.repository, w.merge, key, cur.Merge)
}
return nil
}
// wrote records a module key the merge wrote.
func (w *mergeWrite) wrote(key string) {
if w != nil && !slices.Contains(w.written, key) {
w.written = append(w.written, key)
}
}
// said is the line of an applied merged message: its repository, commit and pull request, and the modules written.
func (w *mergeWrite) said(number int) string {
if w.late != "" {
return fmt.Sprintf("%s at %.12s (#%d): merged message applied as late: older than the taken %.12s, nothing written", w.repository, w.merge, number, w.late)
}
written := "none"
if len(w.written) > 0 {
written = strings.Join(slices.Sorted(slices.Values(w.written)), ", ")
}
return fmt.Sprintf("%s at %.12s (#%d): merged message applied; modules written: %s", w.repository, w.merge, number, written)
}
@@ -0,0 +1,123 @@
package index
import (
"context"
"errors"
"fmt"
"maps"
"slices"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// Placement of modules on nodes (novox/hq ADR 0549 §2, issue 1126): at each merge of the catalogue the forge bridge
// carries every placement file of the merge commit, and the indexer writes each node's key `placement.<node>` in its
// state `placement` only when it differs (the revision alone is no change), deletes the key of a node whose file is
// gone, and says `placement-changed` for each node it wrote or deleted, naming the modules added, removed or changed.
// A merge that changes no placement writes nothing and says nothing.
//
// A file that does not read as a placement, or names another node than its file, is said in the log and its key is left
// as it was: the catalogue's own check refuses such a file before it merges (ADR 0549 §1), so it is a fault of that
// check, and a node keeps composing from the placement it had rather than from none.
// applyPlacement takes the placement files of a merged message in; a message that read none (another repository, an
// older bridge) changes nothing.
func (ix *Indexer) applyPlacement(ctx context.Context, f delivery.Forge) error {
if ix.Placement == nil || f.Kind != delivery.ForgeMerged || !f.PlacementsRead {
return nil
}
keys, err := ix.Placement.Keys(ctx)
if errors.Is(err, process.ErrNotYet) {
// The state is not on the bus yet: said, and the merge's candidates are still taken out (issue 1790's binding
// writes the next catalogue merge's placement once it appears).
ix.logf("%s at %s: its placement files are not written: %v", f.Repository, f.Merge, err)
return nil
}
if err != nil {
return fmt.Errorf("reading the placement keys: %w", err)
}
gone := map[string]bool{}
for _, key := range keys {
if node, err := delivery.ParsePlacementKey(key); err == nil {
gone[node] = true
}
}
var refused []string
for _, node := range slices.Sorted(maps.Keys(f.Placements)) {
delete(gone, node)
next, err := delivery.DecodePlacement([]byte(f.Placements[node]))
if err == nil && next.Node != node {
err = fmt.Errorf("the file of %s names the node %q", node, next.Node)
}
if err != nil {
refused = append(refused, node)
ix.logf("%s at %s: the placement file of %s is refused and placement.%s is left as it was (the catalogue's check should have refused it): %v",
f.Repository, f.Merge, node, node, err)
continue
}
next.Revision = f.Merge
if err := ix.writePlacement(ctx, node, &next); err != nil {
return err
}
}
for _, node := range slices.Sorted(maps.Keys(gone)) {
if err := ix.writePlacement(ctx, node, nil); err != nil {
return err
}
}
if len(refused) > 0 {
// Said, and the merge's candidates are still taken out: a refused file holds nobody's delivery.
ix.logf("%s at %s: the placement files of %v do not read; their keys are left as they were", f.Repository, f.Merge, refused)
}
return nil
}
// writePlacement writes one node's key, or deletes it for next nil, only when it differs, and says what changed.
func (ix *Indexer) writePlacement(ctx context.Context, node string, next *delivery.Placement) error {
key, err := delivery.PlacementKey(node)
if err != nil {
return err
}
var change *delivery.PlacementChange
err = write(ctx, ix.Placement, key, func(value []byte, found bool) ([]byte, bool, error) {
change = nil
var before delivery.Placement
if found {
cur, err := delivery.DecodePlacement(value)
if err != nil {
ix.logf("placement.%s holds what is not a placement, written again: %v", node, err)
} else {
before = cur
}
}
if next == nil {
if !found {
return nil, false, nil
}
c := delivery.ChangeOf(before, delivery.Placement{Node: node})
change = &c
return nil, true, nil
}
if found && delivery.SamePlacement(before, *next) {
return nil, false, nil
}
data, err := delivery.EncodePlacement(*next)
if err != nil {
return nil, false, err
}
c := delivery.ChangeOf(before, *next)
change = &c
return data, false, nil
})
if err != nil || change == nil {
return err
}
if ix.Say != nil {
if err := ix.Say(delivery.PlacementChangedEvent, *change); err != nil {
// The key is written: a reader that watches it hears the change; the event is said loudly as not said.
ix.logf("placement.%s is written, and %s could not be said: %v", node, delivery.PlacementChangedEvent, err)
}
}
return nil
}
@@ -0,0 +1,302 @@
package index
import (
"context"
"errors"
"fmt"
"maps"
"slices"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
// Placement candidates (issue 1126; novox/hq ADR 0549 §4: a placement change passes the composition check before any
// node sees it). An updated pull request of the catalogue that touches placement/ carries every placement file at its
// head (the forge bridge, PlacementsRead). For each node whose file at the head differs from the trunk's key
// `placement.<node>` (SamePlacement, the revision left out), or whose file the head removes, the indexer writes the key
// `<group>.<node>` of its state `placement-candidates`; a node whose file is the trunk's has none, and a key the head no
// longer needs is deleted. The key is written only when its value changes, so its write is the node's event to compose
// in shadow and answer. When the pull request closes or merges, or an open message no longer names it, its keys are
// deleted, and so are they when a head the forge says the changes of (DirsKnown) carries no placement: it touches none.
// A pull message that says nothing of its head (a resend of the open pull requests) leaves them as they are.
//
// A file at the head that does not read as a placement, or names another node, is said in the log and that node gets no
// key (one it had is deleted): the catalogue's own check refuses the pull request (ADR 0549 §1), and a node answers no
// candidate that is not one.
// applyPlacementCandidates takes one forge message into the placement candidates.
func (ix *Indexer) applyPlacementCandidates(ctx context.Context, f delivery.Forge) error {
if ix.PlacementCandidates == nil || !delivery.SameRepository(f.Repository, delivery.CatalogueRepository) || trunkLike[f.Branch] {
return nil
}
if f.Kind == delivery.ForgeMerged || f.Action == delivery.PullClosed {
return ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number
}, fmt.Sprintf("%s#%d %s", f.Repository, f.Number, map[bool]string{true: "merged", false: "closed"}[f.Kind == delivery.ForgeMerged]))
}
if f.Kind != delivery.ForgePull {
return nil
}
if !f.PlacementsRead {
if f.DirsKnown {
// The forge said what this head changes (its directories and paths) and the bridge read no placement: the head
// changes no placement file, so a key an earlier head left goes (ADR 0559 §3).
return ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number
}, fmt.Sprintf("%s#%d at %s changes no placement file", f.Repository, f.Number, f.Head))
}
return nil
}
trunk, err := ix.trunkPlacements(ctx)
if err != nil {
ix.logf("%s#%d at %s: its placement candidates are not written: %v", f.Repository, f.Number, f.Head, err)
return nil
}
nodes := map[string]bool{}
for node := range f.Placements {
nodes[node] = true
}
for node := range trunk {
nodes[node] = true
}
pull := delivery.Head{Repository: f.Repository, Number: f.Number, Head: f.Head}
for _, node := range slices.Sorted(maps.Keys(nodes)) {
key, err := delivery.PlacementCandidateKey(f.Branch, node)
if err != nil {
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
}
var next *delivery.PlacementCandidate // nil: no key for this node
raw, atHead := f.Placements[node]
before, inTrunk := trunk[node]
switch {
case atHead:
p, err := delivery.DecodePlacement([]byte(raw))
if err == nil && p.Node != node {
err = fmt.Errorf("the file of %s names the node %q", node, p.Node)
}
if err != nil {
ix.logf("%s#%d at %s: the placement file of %s is refused, so %s has no placement candidate (the catalogue's check refuses the pull request): %v",
f.Repository, f.Number, f.Head, node, node, err)
break
}
if inTrunk && delivery.SamePlacement(before, p) {
break
}
next = &delivery.PlacementCandidate{Group: f.Branch, Pull: pull, Placement: &p}
case inTrunk:
next = &delivery.PlacementCandidate{Group: f.Branch, Pull: pull} // the head removes the node's file
}
if err := ix.writePlacementCandidate(ctx, key, next); err != nil {
return err
}
}
// A node the head and the trunk both no longer name keeps no key of this group.
return ix.dropPlacementCandidates(ctx, func(node string, c delivery.PlacementCandidate) bool {
return c.Group == f.Branch && delivery.SameRepository(c.Pull.Repository, f.Repository) && c.Pull.Number == f.Number && !nodes[node]
}, fmt.Sprintf("%s#%d at %s", f.Repository, f.Number, f.Head))
}
// trunkPlacements is every node's placement in the trunk's state placement, by node.
func (ix *Indexer) trunkPlacements(ctx context.Context) (map[string]delivery.Placement, error) {
out := map[string]delivery.Placement{}
if ix.Placement == nil {
return nil, errors.New("the indexer reads no placement state")
}
keys, err := ix.Placement.Keys(ctx)
if err != nil {
return nil, fmt.Errorf("reading the placement keys: %w", err)
}
for _, key := range keys {
node, err := delivery.ParsePlacementKey(key)
if err != nil {
continue
}
value, _, found, err := ix.Placement.Get(ctx, key)
if err != nil {
return nil, fmt.Errorf("reading %s: %w", key, err)
}
if !found {
continue
}
p, err := delivery.DecodePlacement(value)
if err != nil {
return nil, fmt.Errorf("%s holds what is not a placement: %w", key, err)
}
out[node] = p
}
return out, nil
}
// writePlacementCandidate writes one candidate key, or deletes it for next nil, only when it changes.
func (ix *Indexer) writePlacementCandidate(ctx context.Context, key string, next *delivery.PlacementCandidate) error {
var data []byte
if next != nil {
var err error
if data, err = delivery.EncodePlacementCandidate(*next); err != nil {
return fmt.Errorf("%w: %s.%s: %v", ErrUnreadable, delivery.PlacementCandidatesState, key, err)
}
}
err := write(ctx, ix.PlacementCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
switch {
case next == nil:
return nil, found, nil
case found && string(value) == string(data):
return nil, false, nil
}
return data, false, nil
})
if errors.Is(err, process.ErrNotYet) {
ix.logf("%s.%s is not written: %v", delivery.PlacementCandidatesState, key, err)
return nil
}
return err
}
// dropPlacementCandidates deletes every candidate key whose node and value gone says, and says each with why.
func (ix *Indexer) dropPlacementCandidates(ctx context.Context, gone func(node string, c delivery.PlacementCandidate) bool, why string) error {
keys, err := ix.PlacementCandidates.Keys(ctx)
if errors.Is(err, process.ErrNotYet) {
return nil
}
if err != nil {
return fmt.Errorf("reading the placement candidate keys: %w", err)
}
for _, key := range keys {
_, node, err := delivery.ParsePlacementCandidateKey(key)
if err != nil {
continue
}
dropped := false
err = write(ctx, ix.PlacementCandidates, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
c, err := delivery.DecodePlacementCandidate(value)
if err != nil {
ix.logf("%s.%s holds what is not a placement candidate, deleted: %v", delivery.PlacementCandidatesState, key, err)
return nil, true, nil
}
dropped = gone(node, c)
return nil, dropped, nil
})
if err != nil {
return err
}
if dropped {
ix.logf("%s.%s is deleted: %s", delivery.PlacementCandidatesState, key, why)
}
}
return nil
}
// PlacementRead is the catalogue's placement as the forge has it now (indexer_placement_reread, ADR 0559 §8): the trunk's
// head and every placement file at it, by node, and each open pull request into the trunk with its own.
type PlacementRead struct {
Trunk string
Files map[string]string
Pulls []PullPlacement
}
// PullPlacement is one open catalogue pull request and every placement file at its head, by node.
type PullPlacement struct {
Branch string
Number int
Head string
Files map[string]string
}
// PlacementReread is the answer of indexer_placement_reread: the heads read, the keys written or deleted in each state,
// and in words what happened.
type PlacementReread struct {
Trunk string `json:"trunk,omitempty"`
Pulls []string `json:"pulls,omitempty"`
Placement []string `json:"placement"`
Candidates []string `json:"candidates"`
Answer string `json:"answer"`
}
// RereadPlacement reads the catalogue's placement now, at the trunk's head and at the head of each open catalogue pull
// request, and writes placement and placement-candidates as a merged and a pull message would (ADR 0549 §2, ADR 0559
// §3), only where a key differs; a candidate key of a pull request that is no longer open, or whose file no longer
// differs at its head, is deleted. It is the recovery for a forge message the indexer missed (ADR 0559 §8).
func (ix *Indexer) RereadPlacement(ctx context.Context) PlacementReread {
out := PlacementReread{Placement: []string{}, Candidates: []string{}}
if ix.ReadPlacement == nil || ix.Placement == nil || ix.PlacementCandidates == nil {
out.Answer = "the indexer reads no placement here: it has no forge reader or no placement state"
return out
}
read, err := ix.ReadPlacement(ctx)
if err != nil {
out.Answer = "the forge could not say the catalogue's placement, nothing is written: " + err.Error()
return out
}
out.Trunk = read.Trunk
before := revisions(ctx, ix.Placement)
beforeC := revisions(ctx, ix.PlacementCandidates)
if err := ix.applyPlacement(ctx, delivery.Forge{Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Merge: read.Trunk,
Placements: read.Files, PlacementsRead: true}); err != nil {
out.Answer = "the trunk's placement could not be written: " + err.Error()
return out
}
open := map[int]bool{}
var failed []string
for _, p := range read.Pulls {
open[p.Number] = true
out.Pulls = append(out.Pulls, fmt.Sprintf("%s#%d at %s", delivery.CatalogueRepository, p.Number, p.Head))
f := delivery.Forge{Kind: delivery.ForgePull, Repository: delivery.CatalogueRepository, Branch: p.Branch, Number: p.Number, Head: p.Head,
Action: delivery.PullUpdated, Placements: p.Files, PlacementsRead: true}
if err := ix.applyPlacementCandidates(ctx, f); err != nil {
failed = append(failed, fmt.Sprintf("#%d: %v", p.Number, err))
}
}
if err := ix.dropPlacementCandidates(ctx, func(_ string, c delivery.PlacementCandidate) bool {
return delivery.SameRepository(c.Pull.Repository, delivery.CatalogueRepository) && !open[c.Pull.Number]
}, "its pull request is not open (indexer_placement_reread)"); err != nil {
failed = append(failed, err.Error())
}
out.Placement = changed(before, revisions(ctx, ix.Placement))
out.Candidates = changed(beforeC, revisions(ctx, ix.PlacementCandidates))
switch {
case len(failed) > 0:
out.Answer = "read at the trunk's head " + read.Trunk + "; some candidates are not written: " + fmt.Sprint(failed)
case len(out.Placement)+len(out.Candidates) == 0:
out.Answer = "read at the trunk's head " + read.Trunk + " and every open catalogue pull request's: every key was as the forge has it, nothing written"
default:
out.Answer = fmt.Sprintf("read at the trunk's head %s and every open catalogue pull request's: %d placement keys and %d candidate keys written or deleted",
read.Trunk, len(out.Placement), len(out.Candidates))
}
return out
}
// revisions is every key of a store with its revision; a state not on the bus yet has none.
func revisions(ctx context.Context, st Store) map[string]uint64 {
out := map[string]uint64{}
keys, err := st.Keys(ctx)
if err != nil {
return out
}
for _, key := range keys {
if _, rev, found, err := st.Get(ctx, key); err == nil && found {
out[key] = rev
}
}
return out
}
// changed is every key written, created or deleted between two revisions maps, sorted.
func changed(before, after map[string]uint64) []string {
out := []string{}
for key, rev := range after {
if before[key] != rev {
out = append(out, key)
}
}
for key := range before {
if _, ok := after[key]; !ok {
out = append(out, key)
}
}
slices.Sort(out)
return out
}
@@ -0,0 +1,201 @@
package index
import (
"context"
"reflect"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0549 §2 (issue 1126): a merge changing one node's file writes that key alone and says one event; a merge with no
// placement change writes nothing and says nothing; a node whose file is gone loses its key and says its modules
// removed; a file that does not read leaves its key; a message that read no placement changes none.
func placementIsWrittenAtAMergeOnlyWhereItChanged(t *testing.T) {
ctx := context.Background()
ix, _, logs := newIndexer(t)
place := newMem()
ix.Placement = place
var said []delivery.PlacementChange
ix.Say = func(event string, body any) error {
if event != delivery.PlacementChangedEvent {
t.Errorf("said %s", event)
}
said = append(said, body.(delivery.PlacementChange))
return nil
}
merged := func(merge string, files map[string]string) delivery.Forge {
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: delivery.CatalogueRepository, Branch: "main",
Number: 7, Head: "h", Merge: merge, Root: "r", Trees: map[string]string{"": "t"}, Placements: files, PlacementsRead: true}
}
ace := `{"v":1,"node":"ace","modules":[{"module":"plex"}]}`
g14 := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`
written := func(node string) (delivery.Placement, uint64) {
t.Helper()
v, ok := place.vals["placement."+node]
if !ok {
return delivery.Placement{}, 0
}
p, err := delivery.DecodePlacement(v)
if err != nil {
t.Fatal(err)
}
return p, place.revs["placement."+node]
}
if err := ix.Apply(ctx, merged("m1", map[string]string{"ace": ace, "g14": g14})); err != nil {
t.Fatal(err)
}
if p, _ := written("ace"); p.Revision != "m1" || !reflect.DeepEqual(p.Assigned(), []string{"plex"}) || len(said) != 2 {
t.Fatalf("first merge: %+v, said %v", p, said)
}
_, aceRev := written("ace")
said = nil
if err := ix.Apply(ctx, merged("m2", map[string]string{"ace": ace, "g14": g14})); err != nil {
t.Fatal(err)
}
if p, rev := written("ace"); rev != aceRev || p.Revision != "m1" || len(said) != 0 {
t.Errorf("a merge with no placement change wrote %+v (revision %d, was %d) and said %v", p, rev, aceRev, said)
}
said = nil
if err := ix.Apply(ctx, merged("m3", map[string]string{"ace": ace, "g14": `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"xterm"}]}`})); err != nil {
t.Fatal(err)
}
if _, rev := written("ace"); rev != aceRev || len(said) != 1 || said[0].Node != "g14" || said[0].Revision != "m3" ||
!reflect.DeepEqual(said[0].Added, []string{"xterm"}) {
t.Errorf("one file changed: ace's revision %d (was %d), said %+v", rev, aceRev, said)
}
said = nil
if err := ix.Apply(ctx, merged("m4", map[string]string{"ace": ace, "g14": `{"v":1,"node":"ace","modules":[]}`})); err != nil {
t.Fatal(err)
}
if p, _ := written("g14"); len(p.Modules) != 2 || len(said) != 0 || !strings.Contains(logs.String(), "the placement file of %s is refused") {
t.Errorf("a file naming another node replaced the key: %+v, said %v, logs %s", p, said, logs)
}
said = nil
if err := ix.Apply(ctx, merged("m5", map[string]string{"ace": ace})); err != nil {
t.Fatal(err)
}
if _, rev := written("g14"); rev != 0 || len(said) != 1 || !reflect.DeepEqual(said[0].Removed, []string{"xterm", "zsh"}) {
t.Errorf("a file removed: g14's key at revision %d, said %+v", rev, said)
}
said = nil
unread := merged("m6", nil)
unread.PlacementsRead = false
if err := ix.Apply(ctx, unread); err != nil {
t.Fatal(err)
}
if p, _ := written("ace"); p.Revision != "m1" || len(said) != 0 {
t.Errorf("a message that read no placement changed it: %+v, %v", p, said)
}
// Issue 1126 (ADR 0549 §4), placement candidates: the trunk now places ace alone. A catalogue pull request writes
// `<group>.<node>` for each node whose file at its head differs from the trunk's, or that the head removes, and only
// when the value changes; a pull message that read no placement, or a pull request changing none, writes nothing; a
// file that does not read gets no key; a merge, a close or an open message not naming it deletes its keys.
cands := newMem()
ix.PlacementCandidates = cands
said = nil
pullOf := func(branch string, number int, head string, files map[string]string) delivery.Forge {
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: delivery.CatalogueRepository, Branch: branch,
Number: number, Head: head, Action: delivery.PullUpdated, Placements: files, PlacementsRead: files != nil}
}
candidate := func(key string) (delivery.PlacementCandidate, uint64, bool) {
t.Helper()
v, ok := cands.vals[key]
if !ok {
return delivery.PlacementCandidate{}, 0, false
}
c, err := delivery.DecodePlacementCandidate(v)
if err != nil {
t.Fatal(err)
}
return c, cands.revs[key], true
}
keys := func() string { k, _ := cands.Keys(ctx); slices.Sort(k); return strings.Join(k, ",") }
g14New := `{"v":1,"node":"g14","modules":[{"module":"zsh"}]}`
for _, row := range []struct {
name string
f delivery.Forge
keys string
}{
{"a pull request adding g14", pullOf("feat/p", 9, h1, map[string]string{"ace": ace, "g14": g14New}), "feat/p.g14"},
{"the same head again", pullOf("feat/p", 9, h1, map[string]string{"ace": ace, "g14": g14New}), "feat/p.g14"},
{"a pull message that read no placement", pullOf("feat/p", 9, h2, nil), "feat/p.g14"},
{"a pull request changing no placement", pullOf("feat/q", 10, h1, map[string]string{"ace": ace}), "feat/p.g14"},
{"a new head removing ace", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
{"a head whose g14 file does not read", pullOf("feat/p", 9, h3, map[string]string{"g14": `{"v":1,"node":"ace","modules":[]}`}), "feat/p.ace"},
{"the head adding g14 again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
{"a head the forge says changes no placement file", func() delivery.Forge { f := pullOf("feat/p", 9, h3, nil); f.DirsKnown = true; return f }(), ""},
{"the head adding g14 and removing ace again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
{"an open message not naming it", delivery.Forge{V: delivery.Version, Kind: delivery.ForgeOpen, Open: []delivery.OpenPull{{Repository: delivery.CatalogueRepository, Number: 10}}}, ""},
{"the pull request heard again", pullOf("feat/p", 9, h2, map[string]string{"g14": g14New}), "feat/p.ace,feat/p.g14"},
{"its merge", func() delivery.Forge {
m := merged("m7", map[string]string{"g14": g14New})
m.Branch, m.Number = "feat/p", 9
return m
}(), ""},
} {
_, revBefore, _ := candidate("feat/p.g14")
if err := ix.Apply(ctx, row.f); err != nil {
t.Fatalf("%s: %v", row.name, err)
}
if got := keys(); got != row.keys {
t.Errorf("%s: keys %q, want %q", row.name, got, row.keys)
}
c, rev, ok := candidate("feat/p.g14")
switch row.name {
case "a pull request adding g14":
if !ok || c.Group != "feat/p" || c.Pull != (delivery.Head{Repository: delivery.CatalogueRepository, Number: 9, Head: h1}) || c.Placement == nil ||
!reflect.DeepEqual(c.Placement.Assigned(), []string{"zsh"}) {
t.Errorf("%s: %+v", row.name, c)
}
case "the same head again", "a pull message that read no placement", "a pull request changing no placement":
if rev != revBefore {
t.Errorf("%s: feat/p.g14 written again (revision %d, was %d)", row.name, rev, revBefore)
}
case "a new head removing ace":
if a, _, _ := candidate("feat/p.ace"); a.Placement != nil || a.Pull.Head != h2 || c.Pull.Head != h2 {
t.Errorf("%s: ace %+v, g14 at %s", row.name, a, c.Pull.Head)
}
}
}
if p, _ := written("g14"); len(p.Modules) != 1 || p.Revision != "m7" {
t.Errorf("the merge's trunk placement of g14: %+v", p)
}
// ADR 0559 §8, indexer_placement_reread: a reread with nothing changed writes nothing; one after a missed merge and a
// missed head writes those keys; one after a missed revert deletes the candidate; a forge that cannot say writes nothing.
g14Other := `{"v":1,"node":"g14","modules":[{"module":"zsh"},{"module":"htop"}]}`
for _, row := range []struct {
name string
read PlacementRead
err error
placement, cands []string
}{
{"nothing changed", PlacementRead{Trunk: "m7", Files: map[string]string{"g14": g14New}}, nil, []string{}, []string{}},
{"a missed merge and a missed head", PlacementRead{Trunk: "m8", Files: map[string]string{"g14": g14New, "ace": ace},
Pulls: []PullPlacement{{Branch: "feat/r", Number: 11, Head: h1, Files: map[string]string{"g14": g14Other, "ace": ace}}}}, nil,
[]string{"placement.ace"}, []string{"feat/r.g14"}},
{"a missed revert", PlacementRead{Trunk: "m8", Files: map[string]string{"g14": g14New, "ace": ace},
Pulls: []PullPlacement{{Branch: "feat/r", Number: 11, Head: h2, Files: map[string]string{"g14": g14New, "ace": ace}}}}, nil,
[]string{}, []string{"feat/r.g14"}},
{"a forge that cannot say", PlacementRead{}, context.DeadlineExceeded, []string{}, []string{}},
} {
ix.ReadPlacement = func(context.Context) (PlacementRead, error) { return row.read, row.err }
r := ix.RereadPlacement(ctx)
if !reflect.DeepEqual(r.Placement, row.placement) || !reflect.DeepEqual(r.Candidates, row.cands) || r.Answer == "" {
t.Errorf("%s: %+v", row.name, r)
}
}
if keys() != "" {
t.Errorf("after the reread of a revert, candidates %q", keys())
}
}
@@ -45,12 +45,6 @@ type taken struct {
}
func (ix *Indexer) indexRegistered(ctx context.Context, r Registration, force bool) (taken, error) {
return ix.indexAt(ctx, r, force, false)
}
// indexAt is indexRegistered; reread also reads a module the modules state holds (a person's take-in of a held module
// whose record stands behind the head the gate's source names, issue 1126).
func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bool) (taken, error) {
if ix.Modules == nil || ix.Read == nil {
return taken{why: "the indexer reads no forge or writes no modules"}, nil
}
@@ -61,7 +55,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
}
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
return taken{}, fmt.Errorf("modules.%s: %w", key, err)
} else if found && !reread {
} else if found {
ix.markRead(r.Module)
return taken{held: true, why: "the modules state holds it already: a merge to its repository writes it from now on"}, nil
}
@@ -129,7 +123,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
if l := read.Locks[dir]; len(l) > 0 {
f.Locks = map[string]map[string]string{dir: l}
}
if _, err := ix.applyManifests(ctx, f, false); err != nil {
if err := ix.applyManifests(ctx, f, false); err != nil {
return t, err
}
if _, _, found, err := ix.Modules.Get(ctx, key); err != nil {
@@ -210,9 +204,6 @@ type TakenIn struct {
// controller's records, or whose every registration could not be read or holds no manifest of it, is answered with why.
func (ix *Indexer) TakeIn(ctx context.Context, module string) TakenIn {
out := TakenIn{Module: module}
if again, ok := ix.takeInHeld(ctx, module); ok {
return again
}
if ix.Builds == nil {
out.Answer = "the indexer reads no controller's build records, so it knows no registration"
return out
@@ -282,69 +273,3 @@ func (ix *Indexer) IndexRegisteredAll(ctx context.Context) error {
}
return first
}
// takeInHeld reads again, at the head the gate's source names for its repository, a module the modules state holds whose
// record stands at another commit, and writes it as a merge would (only when it changed). It is the recovery for a merge
// whose merged message never reached the indexer (issue 1126: mesh-catalog#535's message was given up by the forge bridge
// after its reads timed out, and mesh-vault's record stayed at the merge before). It answers false where it does not
// apply: no such module held, or no head named, or the record at that head already.
func (ix *Indexer) takeInHeld(ctx context.Context, module string) (TakenIn, bool) {
out := TakenIn{Module: module}
key, err := delivery.ModulesKey(module)
if err != nil || ix.Modules == nil {
return out, false
}
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil || !found {
return out, false
}
cur, err := delivery.DecodeModule(value)
if err != nil || cur.Repository == "" || cur.Path == "" {
return out, false
}
head := ""
ix.mu.Lock()
for repo, h := range ix.source.heads {
if delivery.SameRepository(repo, cur.Repository) && h != "" {
head = h
}
}
ix.mu.Unlock()
if head == "" || head == cur.Commit {
return out, false
}
// Never backwards (order.go): a source head older than the record (a merge applied before the source moved) is never
// read and written back over it. A diverged or rewritten history, or a record whose commit the forge no longer has, is
// not older and is read; where the forge cannot say otherwise, nothing is read. One forge call.
old, why := true, "the forge cannot be asked"
if ix.Contains != nil {
var err error
if old, err = ix.older(ctx, cur.Repository, head, cur.Commit); err != nil {
old, why = true, err.Error()
} else if old {
why = fmt.Sprintf("%.12s is older than its record at %.12s", head, cur.Commit)
}
}
if old {
out.Held, out.Repository, out.Commit, out.Path = true, cur.Repository, cur.Commit, cur.Path
out.Answer = fmt.Sprintf("held; the source names another head, %.12s, and not one newer than its record (%s), so it is not read again", head, why)
return out, true
}
t, err := ix.indexAt(ctx, Registration{Module: module, Repository: cur.Repository, Path: cur.Path, Commit: head}, true, true)
out.Held, out.Repository, out.Commit, out.Path = true, t.repository, t.commit, pathOf(t.dir)
switch {
case err != nil:
out.Answer = "held, and not read again at the source's head: " + err.Error()
case !t.stored:
out.Answer = "held, and not read again at the source's head: " + t.why
default:
after, _, _, _ := ix.Modules.Get(ctx, key)
out.Stored = true
if string(after) == string(value) {
out.Answer = fmt.Sprintf("held, and read again from %s at %s: unchanged there, so its record stands", t.repository, t.commit)
} else {
out.Answer = fmt.Sprintf("held, read again from %s at %s and written to the modules state: it changed since %.12s", t.repository, t.commit, cur.Commit)
}
}
return out, true
}

Some files were not shown because too many files have changed in this diff Show More