Compare commits

..
Author SHA1 Message Date
jschoubben 83f7a7612d Review round 2: a held module's object reads come from its running manifest; one with none keeps the list in force, said; indexer_judge asks the engine for the running build again (issue 774)
mesh/delivery-group group feat/l3a-waits rejected: composed together: the checker's verdict at the members' heads (de2cdacb9cde): shanks
mesh/delivery closed: the pull request closed unmerged
mesh/check failed (3): shanks: novox/mesh-sdk#163 does not merge into the trunk: files in conflict: go/deli…; and 2 more | 83f7a7612d19 on c4470f42dba9
2026-11-03 13:07:11 +01:00
jschoubben 1fe1beee58 Review round 1: the indexer learns its build from the node-engine's seat; a held module's seats and hearings come from its running manifest; indexer_judge; one judge pass at a time (issue 774) 2026-11-03 13:03:11 +01:00
jschoubben 40da5e924e A manifest with a field the judge does not know waits instead of being refused; a held module keeps its bus users; tools list and redo the verdicts (ADR 0578 amended, issue 774) 2026-11-03 12:53:07 +01:00
222 changed files with 1035 additions and 15751 deletions
@@ -27,7 +27,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/health"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provisions"
"mesh-delivery-assigner/internal/judge"
"mesh-delivery-assigner/internal/assign"
@@ -435,61 +434,6 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
if !self {
t.Error("ace's own want does not say it composes its own once placed")
}
// ADR 0603 (L3, D15): one source revision changes a provider (db), its new consumer (app) and a build module (tool, whose
// recipe moved: ace runs its first build and the second, under the agreed toolchain, is the one the source revision names).
// The keys are written as their writers write them: the indexer's modules and manifests, the builder's builds, the
// controller's grants, the placement seat's placement and the provisioner's provisions. The build module's want moves at
// once and the provider's with it; the consumer's waits for its provision, with the words naming it, and moves once the
// provisioner provides it (rule 8, ADR 0570 §4): nothing waits for a turn or an order, only for a key.
offer := `"provides":[{"name":"database","scope":"mesh","answered-by":"helper"}]`
dbDigest := b.manifest(t, "db", `{"module":"db","version":"1",`+offer+`}`)
appDigest := b.manifest(t, "app", `{"module":"app","version":"1","requires":["database"]}`)
toolDigest := b.manifest(t, "tool", `{"module":"tool","version":"1"}`)
asked := func(state, why string) {
raw, err := provisions.Encode(provisions.Provision{V: provisions.Version, Provision: "database", Consumer: provisions.End{Module: "app", Node: "ace"},
Provider: provisions.End{Module: "db", Node: "ace"}, State: state, Why: why, At: time.Now()})
if err != nil {
t.Fatal(err)
}
b.putJSON(t, provisions.ProvisionsBucket, "ace.app.database", raw)
}
// The provisioner keys it from the manifests, as soon as they are known and whether the provider is placed yet or not.
asked(provisions.Asked, "the provider's build is not done")
fpOf := func(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
toolOld := fpOf('1')
for module, row := range map[string]struct {
fp, digest string
}{"db": {fpOf('2'), dbDigest}, "app": {fpOf('3'), appDigest}, "tool": {fpOf('4'), toolDigest}} {
mod, _ := delivery.EncodeModule(delivery.Module{FP2: row.fp, Manifest: row.digest, Repository: "novox/" + module, Commit: strings.Repeat("2", 40), Path: delivery.RootPath})
b.putJSON(t, delivery.ModulesBucket, module, mod)
fps := []string{row.fp}
if module == "tool" {
fps = append(fps, toolOld)
}
for _, fp := range fps {
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: row.digest})
bkey, _ := delivery.BuilderBuildKey(module, fp)
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
gkey, _ := delivery.PartGrantKey(module, fp)
grant, _ := delivery.EncodeGrant(delivery.Grant{})
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
}
b.place(t, "ace", module)
}
runsOld, _ := delivery.EncodeHave(delivery.Have{FP: toolOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
b.putJSON(t, delivery.HaveBucket, "ace.tool", runsOld)
if w := b.wantNow(t, "ace", "tool", "the build module's want at the second build", func(w delivery.Want) bool { return w.FP == fpOf('4') }); w.FP != fpOf('4') {
t.Errorf("tool's want = %+v", w)
}
b.wantNow(t, "ace", "db", "the provider's want at its fingerprint", func(w delivery.Want) bool { return w.FP == fpOf('2') })
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.ace.app"); err == nil {
t.Errorf("app was wanted while its provision was asked: %s", e.Value())
}
asked(provisions.Provided, "")
if w := b.wantNow(t, "ace", "app", "the consumer's want once its provision is provided", func(w delivery.Want) bool { return w.FP == fpOf('3') }); w.FP != fpOf('3') {
t.Errorf("app's want = %+v", w)
}
}
// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
@@ -28,8 +28,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"mesh-delivery-assigner/internal/assign"
)
@@ -62,9 +60,6 @@ var readBuckets = []string{
delivery.PresenceBucket,
delivery.HaveBucket,
provisions.ProvisionsBucket,
delivery.SourceBucket,
delivery.MergesBucket,
graph.Bucket,
}
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
@@ -145,17 +140,6 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
continue
}
if (name == delivery.SourceBucket || name == delivery.MergesBucket) && err != nil {
// The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of
// a record module is asked of a person, as before.
logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err)
continue
}
if name == graph.Bucket && err != nil {
// The indexer's module graph (ADR 0614 §2, §3): without it no want waits for a verb it invokes until it is bound.
logf("the bucket %s cannot be opened (%v): no want waits for a verb it invokes until it is bound once the bus says it was made", name, err)
continue
}
if name == delivery.PartGrantsBucket && err != nil {
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
@@ -174,7 +158,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
func startsWithout(name string) bool {
switch name {
case delivery.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket, graph.Bucket:
delivery.AssignerPutsBucket:
return true
}
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
@@ -257,9 +241,6 @@ var watched = []string{
assign.PoliciesBucket,
provisions.ProvisionsBucket,
delivery.SeatHoldersBucket,
delivery.SourceBucket,
delivery.MergesBucket,
graph.Bucket,
}
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
@@ -355,9 +336,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
started++
var opts []jetstream.WatchOpt
if name == delivery.HaveBucket || name == delivery.SourceBucket {
// what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1)
opts = append(opts, jetstream.IncludeHistory())
if name == delivery.HaveBucket {
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
}
if err := watch(name, opts...); err != nil {
return err
@@ -155,14 +155,10 @@ func (p *passer) wants(ctx context.Context) {
defer func() { p.sayNeither(neither, stale, left) }()
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
grants := p.w.grantsBucket()
// The verb edges, once for the pass (ADR 0614 §2, §3).
moduleGraph := p.moduleGraph()
served := p.served(ctx, get, moduleGraph, nodes)
offering := p.offering(ctx, get, manifests, nodes) // bus state (ADR 0614 §5)
for name := range names {
fp := trunk[name].FP2
name := name
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
Self: func(node string) bool { return placed[node] },
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
@@ -178,8 +174,7 @@ func (p *passer) wants(ctx context.Context) {
Allowed: func(node, digest string) bool {
key, err := delivery.RecordAskKey(node, name, digest)
return err == nil && p.answeredWith(key, assign.OptionYes)
},
Release: func(fp string) (string, bool) { return p.w.release(name, fp) }}
}}
if fp != "" {
held, _ := delivery.HeldKey(name, fp)
grant, _ := delivery.PartGrantKey(name, fp)
@@ -196,10 +191,6 @@ func (p *passer) wants(ctx context.Context) {
p.clearHeld(name, fp)
}
}
if served != nil {
m.Invokes, m.Served = keptInvokes(moduleGraph, name), served
}
m.Offering = offering
last := map[string]string{}
for _, node := range m.Nodes {
last[node] = p.w.lastHealthy(node, name, fp)
@@ -828,9 +819,7 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
// person must allow (ADR 0547 §3): for a have of phase
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. A
// move of the build alone (the have says so) to a build an approved release carries is given its yes by that Approve, the
// want naming the release, with no ask (ADR 0601 §1).
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again.
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) {
for _, node := range m.Nodes {
if m.Self == nil || !m.Self(node) {
@@ -847,37 +836,25 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo
if w.Removed {
continue
}
yes, release := "", ""
yes := ""
if had && was.FP == w.FP {
yes, release = was.Yes, was.Release
yes = was.Yes
}
key, err := delivery.HaveKey(node, module)
if err != nil {
continue
}
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
switch {
case m.Allowed != nil && m.Allowed(node, h.Waits):
yes, release = h.Waits, ""
case h.BuildAlone && h.FP != w.FP && m.Release != nil:
// A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1),
// named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked.
if r, ok := m.Release(w.FP); ok {
yes, release = h.Waits, r
break
}
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
default:
if m.Allowed != nil && m.Allowed(node, h.Waits) {
yes = h.Waits
} else {
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
}
}
// A want carried over from before the node composed its own may still name the controller's composition; a placed
// node's want names the build alone (ADR 0547 §2).
if yes == "" {
release = ""
}
if yes != w.Yes || release != w.Release || w.Composed != "" || !w.Self {
w.Yes, w.Release, w.Composed, w.Self = yes, release, "", true
if yes != w.Yes || w.Composed != "" || !w.Self {
w.Yes, w.Composed, w.Self = yes, "", true
out.Wants[node] = w
}
}
@@ -1094,26 +1071,6 @@ func (p *passer) seatHolders(ctx context.Context) {
}
}
// offered says whether some manifest of the trunk, or a module the mesh provides itself, offers a provision under its name
// (a module offers its own name too, catalogue.Manifest.Offers): what a requirement nobody offers is told from.
func offered(manifests map[string]catalogue.Manifest) func(string) bool {
names := map[string]bool{}
add := func(m catalogue.Manifest) {
for _, o := range m.Offers() {
names[o] = true
}
}
for _, m := range manifests {
add(m)
}
for _, module := range delivery.ProvidedModules() {
if m, _, err := delivery.ProvidedManifest(module); err == nil {
add(m)
}
}
return func(provision string) bool { return names[provision] }
}
// answering is, by provision, the manifests of the modules placed on some node that provide it: what a want's
// provisions are judged by (catalogue.Awaited, issue 774). A provider in the catalogue placed nowhere answers nobody.
func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(string) []catalogue.Manifest {
@@ -17,8 +17,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"mesh-delivery-assigner/internal/assign"
)
@@ -113,14 +111,6 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
// taken away, is not taken for the hold that stands now.
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
// ADR 0603 §5: a provision is offered by a manifest's provides, by a module's own name, and by a module the mesh provides
// itself (mesh-addressing, mesh-wireguard's); nothing else.
off := offered(map[string]catalogue.Manifest{"node-tools": {Module: "node-tools"}, "db": {Module: "db", Provides: []catalogue.Offer{{Name: "database"}}}})
for provision, want := range map[string]bool{"node-tools": true, "database": true, "mesh-addressing": true, "queue": false} {
if off(provision) != want {
t.Errorf("offered(%s) = %v, want %v", provision, !want, want)
}
}
key, _ := delivery.HoldAskKey("web", fpTest)
p := newPasser(buckets{}, time.Now())
put := func(asking int) string {
@@ -150,23 +140,6 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
t.Error("an answer to an ask taken away was taken")
}
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
// ADR 0614 §4: the rules read what each node's engine states it reads from its record in `<node>.node`; a node with no
// record, or one that states nothing, is read as stating nothing (the rules then read the baseline).
t.Run("the rules read what each node's engine reads", func(t *testing.T) {
w := newWorld()
for node, reads := range map[string][]string{"ace": {"word:module", "word:tool-runner"}, "g14": nil} {
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, ManifestReads: reads}})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveNodeKey(node)
w.put(delivery.HaveBucket, key, raw)
}
n := w.nodes(nil, nil)
if !slices.Equal(n.Reads["ace"], []string{"word:module", "word:tool-runner"}) || len(n.Reads["g14"]) != 0 || len(n.Reads["novox"]) != 0 {
t.Errorf("the nodes' reads are %v", n.Reads)
}
})
}
// memEntry is a value a memKV holds, as a bucket's Get answers it.
@@ -407,8 +380,6 @@ func putOfTheBus(t *testing.T) {
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
aPlacementAloneNamesWhereModulesRun(t)
// The verb edges a want waits on come from the graph and what runs (ADR 0614 §2, §3): folded here to hold the count.
aWantWaitsOnTheVerbEdgesTheGraphKeeps(t)
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
var said []asks.Condition
@@ -443,11 +414,10 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
t.Errorf("the clear said %q", last)
}
// 1945: a module that leaves neither list with no want is cleared in words saying so, never "has a want again". The bus is
// on neither list by a rule as any module (ADR 0601 §1), nothing for the operator to do.
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Rule: delivery.RuleGrant, Words: "no credential"}}}
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
p.sayNeither(bus, nil, nil)
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NothingToDo) || !strings.Contains(c.Explanation, "rule 6 holds its first want back: no credential") {
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
t.Errorf("the bus's neither condition = %+v", c)
}
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
@@ -504,6 +474,7 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
{nil, wanted, "rofi on shanks has a want"},
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
} {
logged = nil
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
@@ -817,74 +788,6 @@ func mustKey(key string, err error) string {
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
aReleasesApproveAnswersTheMoveOfItsBuildAlone(t)
}
// ADR 0601 §1: a placed node's have waiting for a yes to a move of the build alone, to a build an approved release carries, is
// given its yes by that Approve, the want naming the release, and no record ask is raised; a move that changes a setting or a
// secret too, one at the build it runs, and one no approved release carries are asked of a person as before.
func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
commit := strings.Repeat("1", 40)
for _, c := range []struct {
name string
alone bool
at string // the fingerprint the have runs
mergedAt string // the commit the release merged web at; "" for no merge of the release
fpSrc uint64 // the source revision at which web's fingerprint last changed
yes, asked bool
}{
{"the build alone, its release approved", true, fpOld, commit, 7, true, false},
{"a setting changed too", false, fpOld, commit, 7, false, true},
{"a setting changed at the build it runs", true, fpTest, commit, 7, false, true},
{"no approved release carries it", true, fpOld, "", 7, false, true},
{"the release merged another commit", true, fpOld, strings.Repeat("2", 40), 7, false, true},
// Its fingerprint last moved at another release's revision, which merged nothing of web: that Approve answers nothing.
{"a fingerprint another release moved", true, fpOld, commit, 8, false, true},
// Review rounds 1 and 2: moved by a recompute after the release (a toolchain, a package, a base) with no merge, the
// indexer writes fp-src 0, not known, though Src is still the release's: asked of a person.
{"a fingerprint moved by a recompute after the release", true, fpOld, commit, 0, false, true},
} {
asksKV := &memKV{values: map[string][]byte{}}
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV}, time.Now())
p.logf = t.Logf
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FPSrc: c.fpSrc, Path: delivery.RootPath})
p.w.put(delivery.ModulesBucket, "web", mod)
src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/web", Heads: map[string]string{"novox/web": commit}})
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, src, 7, time.Now())
later, _ := delivery.EncodeSource(delivery.Source{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}})
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release
if c.mergedAt != "" {
merge, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []delivery.MergePull{
{Head: delivery.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)}, State: delivery.MergeMerged, Commit: c.mergedAt}}})
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.MergesBucket, "feat/web", merge)
other, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/other", Ask: "merge.feat_other.0123456789ab", Pulls: []delivery.MergePull{
{Head: delivery.Head{Repository: "novox/x", Number: 4, Head: strings.Repeat("4", 40)}, State: delivery.MergeMerged, Commit: commit}}})
if err != nil {
t.Fatal(err)
}
p.w.put(delivery.MergesBucket, "feat/other", other)
}
have, _ := delivery.EncodeHave(delivery.Have{FP: c.at, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone})
p.w.put(delivery.HaveBucket, "g14.web", have)
m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true },
Release: func(fp string) (string, bool) { return p.w.release("web", fp) }}
out := assign.Outcome{Wants: map[string]delivery.Want{"g14": {FP: fpTest, Self: true}}}
p.yeses(context.Background(), "web", m, map[string]delivery.Want{"g14": {FP: fpOld, Self: true}}, &out)
w := out.Wants["g14"]
if got := w.Yes == waiting && w.Release == "feat/web"; got != c.yes || (!c.yes && (w.Yes != "" || w.Release != "")) {
t.Errorf("%s: the want is %+v", c.name, w)
}
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
if _, there := asksKV.values[akey]; there != c.asked {
t.Errorf("%s: a record ask raised %v, want %v", c.name, there, c.asked)
}
}
}
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
@@ -1025,61 +928,3 @@ func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
t.Fatalf("the old conditions cleared = %+v", conds)
}
}
// ADR 0614 §2, §3: of what a module invokes, the pass keeps as edges the verbs the graph does not leave out and another
// module serves; a verb is served once a running healthy build's manifest (the builder's build, the indexer's manifest)
// serves it, and a build whose manifest cannot be read counts what the graph says its module serves. No graph read: no wait.
func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
p := newPasser(buckets{}, time.Now())
fp := func(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
if p.served(context.Background(), nil, graph.Graph{}, assign.Nodes{}) != nil {
t.Error("with no graph read, a want waits on a verb")
}
for module, n := range map[string]graph.Node{
"caller": {FP: fp("c"), Repository: "novox/caller", Invokes: []string{"seat:x.v", "loop.l", "none.t", "caller.own"}, LeftOut: []string{"loop.l"},
Serves: []string{"caller.own"}},
"server": {FP: fp("e"), Repository: "novox/server", Serves: []string{"seat:x.v", "seat:x.w"}},
"loop": {FP: fp("1"), Repository: "novox/loop", Serves: []string{"loop.l"}},
"late": {FP: fp("2"), Repository: "novox/late", Serves: []string{"late.t"}},
} {
raw, err := graph.Encode(n)
if err != nil {
t.Fatal(err)
}
p.w.put(graph.Bucket, module, raw)
}
g := p.moduleGraph()
if got := keptInvokes(g, "caller"); !slices.Equal(got, []string{"seat:x.v"}) || keptInvokes(g, "unknown") != nil {
t.Errorf("the kept verb edges of caller = %v", got)
}
digest := strings.Repeat("d", 64)
man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "server", JSON: `{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`})
get := func(_ context.Context, key string) ([]byte, error) {
if key == digest {
return man, nil
}
return nil, errNotThere
}
build, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
key, _ := delivery.BuilderBuildKey("server", fp("f"))
p.w.put(delivery.BuilderBuildsBucket, key, build)
running := delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
at := func(f string) delivery.Have { h := running; h.FP = f; return h }
served := p.served(context.Background(), get, g, assign.Nodes{Haves: map[string]map[string]delivery.Have{
"g14": {"server": at(fp("f")), "late": at(fp("2"))}}})
for v, want := range map[string]bool{"seat:x.w": true, "seat:x.v": false, "late.t": true} {
if served(v) != want {
t.Errorf("served(%s) = %v, want %v: the running server's manifest serves w alone, late's is not read", v, served(v), want)
}
}
// ADR 0614 §5: bus-state counts as offered only from a healthy build on a node present; the bus module's build on an
// offline node offers nothing, and one whose manifest is not read counts its trunk manifest's offer.
bus := map[string]catalogue.Manifest{"bus": {Module: "bus", Provides: []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}}}
for _, online := range []bool{false, true} {
offering := p.offering(context.Background(), get, bus, assign.Nodes{Online: map[string]bool{"n2": online},
Haves: map[string]map[string]delivery.Have{"n2": {"bus": at(fp("2"))}}})
if offering(graph.BusState) != online {
t.Errorf("with the bus module's node online %v, bus-state offered %v", online, offering(graph.BusState))
}
}
}
@@ -385,7 +385,7 @@ func wantsVerbs(t *testing.T) {
}
manifest("web", "")
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}],"upgrade":{"policy":"record","why":"its streams"}`)
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}]`)
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
}
@@ -406,7 +406,7 @@ func wantsVerbs(t *testing.T) {
if p := policyOf("photos"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule {
t.Errorf("photos = %+v", p)
}
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule { // the policy its manifest says (issue 774)
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus {
t.Errorf("nats = %+v", p)
}
// compare: each node's own composition is the digest its have names, and the assigner's findings; nothing splits, the
@@ -471,12 +471,8 @@ func wantsVerbs(t *testing.T) {
len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") {
t.Errorf("chosen %+v, asked %v, said %v", c, asked, said)
}
// A person may choose the bus's policy as any module's (ADR 0601 §1: no part treats the module that provides the bus apart).
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err != nil {
t.Errorf("a person's policy for the bus was refused: %v", err)
}
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "default", "why": "back to the module"}, "op"); err != nil {
t.Fatal(err)
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err == nil {
t.Error("the bus was given a policy")
}
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "roll"}, "op"); err == nil {
t.Error("a policy without why was set")
@@ -1,128 +0,0 @@
package main
import (
"context"
"slices"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"mesh-delivery-assigner/internal/assign"
)
// The verb edges a want waits on (novox/hq ADR 0614 §2, §3): the module graph the indexer writes names what each module
// invokes and serves and which of its verb edges it leaves out on a cycle (one place decides the cycle); a want waits for
// each kept edge until a build serving its verb runs healthy, read from the haves, the builder's builds and the indexer's
// manifests, as a provider is counted (ADR 0603 §5).
// moduleGraph is the module graph as the indexer last wrote it; a key that does not decode is left out, so its module
// waits on no verb.
func (p *passer) moduleGraph() graph.Graph {
g := graph.Graph{}
for key, raw := range p.w.read(graph.Bucket) {
module, err := graph.ParseKey(key)
if err != nil {
continue
}
if n, err := graph.Decode(raw); err == nil {
g[module] = n
}
}
return g
}
// keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module
// other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds
// no node of module.
func keptInvokes(g graph.Graph, module string) []string {
n, ok := g[module]
if !ok {
return nil
}
var out []string
for _, v := range n.Invokes {
if slices.Contains(n.LeftOut, v) {
continue
}
if slices.ContainsFunc(g.Servers(v), func(s string) bool { return s != module }) {
out = append(out, v)
}
}
return out
}
// served is, for this pass, whether a build serving a verb runs healthy on some node (graph.Served): every running healthy
// have's build (the builder's builds key of its module and fingerprint) names its manifest, and the verbs that manifest
// serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in
// the log, so a build read late never holds its callers for good; nil when the graph holds nothing, and no want waits on
// a verb.
func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign.Nodes) func(string) bool {
if len(g) == 0 {
return nil
}
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) },
func(module string) []string { return g[module].Serves })
if len(unread) > 0 {
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
}
return served
}
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A
// healthy build whose manifest cannot be read counts what its module's trunk manifest offers, so a build read late never
// holds a consumer for good.
func (p *passer) offering(ctx context.Context, get getter, trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool {
offered := map[string]bool{}
for node, haves := range n.Haves {
if !n.Online[node] {
continue
}
for module, h := range haves {
if h.Phase != delivery.PhaseRunning || h.Health != delivery.HealthOK {
continue
}
m, ok := p.manifestOf(ctx, get, module, h.FP)
if !ok {
m = trunk[module]
}
for _, o := range m.Offers() {
offered[o] = true
}
}
}
return func(provision string) bool { return offered[provision] }
}
// servesOf are the verbs the manifest of module's build at fp serves (graph.ServesOf), and whether it could be read.
func (p *passer) servesOf(ctx context.Context, get getter, module, fp string) ([]string, bool) {
m, ok := p.manifestOf(ctx, get, module, fp)
if !ok {
return nil, false
}
return graph.ServesOf(m), true
}
// manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests),
// and whether it could be read.
func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string) (catalogue.Manifest, bool) {
key, err := delivery.BuilderBuildKey(module, fp)
if err != nil {
return catalogue.Manifest{}, false
}
raw := p.w.value(delivery.BuilderBuildsBucket, key)
if raw == nil {
return catalogue.Manifest{}, false
}
b, err := delivery.DecodeBuilderBuild(raw)
if err != nil || b.Manifest == "" {
return catalogue.Manifest{}, false
}
got := p.w.manifest(ctx, get, b.Manifest)
if got.err != "" {
return catalogue.Manifest{}, false
}
return got.manifest, true
}
@@ -197,14 +197,18 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
}
// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log
// and says policy-changed. The bus's choice is a person's as any module's (ADR 0601 §1).
// and says policy-changed. The bus is never rolled: a choice for it is refused.
func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) {
if module == "" {
return nil, errors.New("a policy is set for one module: name it")
}
if _, ok := manifests[module]; !ok {
m, ok := manifests[module]
if !ok {
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
}
if catalogue.ProvidesBus(m) {
return nil, fmt.Errorf("%s is the bus, which is never rolled whoever says so (novox/hq ADR 0236): its step is the put on the operator's warrant", module)
}
if why == "" {
return nil, errors.New("a person's policy says why")
}
@@ -37,9 +37,6 @@ type world struct {
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
// delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner.
placement *delivery.PlacementKeys
// sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its
// history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1).
sourceGroups map[uint64]string
}
// entryMeta is a value's revision in its bucket and when it was written.
@@ -57,8 +54,7 @@ type parsed struct {
func newWorld() *world {
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys(),
sourceGroups: map[uint64]string{}}
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys()}
}
// putEntry is put, keeping the value's revision and the time it was written.
@@ -74,44 +70,6 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t
if bucket == delivery.PlacementBucket {
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
}
if bucket == delivery.SourceBucket && key == delivery.SourceMainKey {
if s, err := delivery.DecodeSource(value); err == nil && s.Group != "" {
w.sourceGroups[revision] = s.Group
}
}
}
// release is the release whose Approve carries module's build fp (ADR 0601 §1), and whether one does: the trunk's value of
// the module names fp, the source revision at which that fingerprint last changed (FPSrc, never the Src of its last merge,
// which a recompute with no merge keeps) names the release that moved it, and the gate's merge of that release, which it
// writes on the operator's Approve, holds the module's own trunk merge among its merged pull requests. Any of them
// unknown: none, and the move is asked of a person.
func (w *world) release(module, fp string) (string, bool) {
t, ok := w.trunk()[module]
if !ok || fp == "" || t.FP2 != fp || t.FPSrc == 0 || t.Commit == "" {
return "", false
}
w.mu.Lock()
group := w.sourceGroups[t.FPSrc]
w.mu.Unlock()
key, err := delivery.MergesKey(group)
if group == "" || err != nil {
return "", false
}
raw := w.value(delivery.MergesBucket, key)
if raw == nil {
return "", false
}
merge, err := delivery.DecodeMerge(raw)
if err != nil {
return "", false
}
for _, pull := range merge.Pulls {
if pull.State == delivery.MergeMerged && pull.Commit == t.Commit && delivery.SameRepository(pull.Repository, t.Repository) {
return group, true
}
}
return "", false
}
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
@@ -338,8 +296,7 @@ func (w *world) wantsOf() map[string]map[string]delivery.Want {
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{},
Reads: map[string][]string{}}
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{}}
for key, value := range w.read(delivery.PresenceBucket) {
node, _, _, err := delivery.ParsePresenceKey(key)
if err != nil || node == "" {
@@ -351,14 +308,7 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
}
for key, value := range w.read(delivery.HaveBucket) {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil {
continue
}
if isNode {
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
n.Reads[node] = h.Record.ManifestReads
}
if err != nil || isNode {
continue
}
if h, err := delivery.DecodeHave(value); err == nil {
+1 -5
View File
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
@@ -20,7 +20,3 @@ require (
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -119,6 +119,14 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
if n := len([]rune(headline)); n > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
if h.Bus {
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline, Needs: needs,
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
asks.NeedsYou, needs, module, node, h.Words)}
}
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
Summary: headline, Headline: headline,
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
@@ -16,7 +16,7 @@ import (
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus as any module (ADR 0601 §1).
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus never rolled.
// PoliciesBucket is the assigner's state `policies`.
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
@@ -1,38 +0,0 @@
package assign
import (
"slices"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
// verbWaiting says why the module's want on node waits for a verb it invokes (novox/hq ADR 0614 §2): the first seat verb or
// module tool of Invokes (the module graph's verb edges, those it leaves out on a cycle not among them, §3) that no
// running healthy build serves. Nil Served (nothing read of what runs) waits for none, and a verb a running build serves
// already adds no wait. It holds under the provisions' rule (RuleProvisions): no reader knows a rule number of its own
// yet, so the words name the edge (graph.VerbWaitWords, which the delivery view reads).
func (m Module) verbWaiting(node string) (string, bool) {
if m.Served == nil {
return "", false
}
for _, v := range slices.Sorted(slices.Values(m.Invokes)) {
if !m.Served(v) {
return graph.VerbWaitWords(v, m.Name, node), true
}
}
return "", false
}
// busStateWaiting says why the module's want on node waits for bus state (ADR 0614 §5): its manifest asks bus state (a
// bus-state entry, as the module graph reads it) and no build whose manifest offers bus-state runs healthy on a node
// present. The provision bus-state is answered with no credential of a consumer's own, so ADR 0603 §5 counts the
// provider's own have and no provisioner key is waited for, handed or not. Nil Offering (nothing read of what runs) waits
// for none, and neither does a module that offers bus-state itself. It holds under the provisions' rule, its words naming bus state (graph.BusStateWaitWords).
func (m Module) busStateWaiting(node string) (string, bool) {
if len(m.Manifest.BusState) == 0 || m.Offering == nil || m.Offering(graph.BusState) ||
slices.Contains(m.Manifest.Offers(), graph.BusState) { // the bus module offering it waits on nobody for it
return "", false
}
return graph.BusStateWaitWords(m.Name, node), true
}
@@ -11,8 +11,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
@@ -38,21 +36,6 @@ type Module struct {
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
Answering func(provision string) []catalogue.Manifest
// Offered says whether any module offers a provision by name or by its own name (catalogue.Manifest.Offers) among every
// manifest the trunk holds and the modules the mesh provides itself, placed or not. Nil: not known, and no requirement
// is waited on for want of an offer.
Offered func(provision string) bool
// Invokes are the seat verbs and module tools the module graph says the module invokes and keeps as edges to the
// modules serving them, those it leaves out on a cycle not among them (ADR 0614 §2, §3): the want waits until each is
// served (Served). Nil: no graph node is read, and no want waits on a verb.
Invokes []string
// Served says whether a build serving a seat verb or module tool runs healthy, counted from its holder's have (ADR 0614
// §2, as ADR 0603 §5 counts a provider's). Nil: nothing is read of what runs, and no want waits on a verb.
Served func(invoke string) bool
// Offering says whether a build whose manifest offers a provision runs healthy on a node present, counted from the
// provider's own have (ADR 0603 §5): what a want asking bus state waits for (ADR 0614 §5). Nil: nothing is read of what
// runs, and no want waits on bus state.
Offering func(provision string) bool
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
Chosen *catalogue.PersonPolicy
Nodes []string
@@ -62,10 +45,6 @@ type Module struct {
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
// with this digest (ADR 0539 §14). Nil: none did.
Allowed func(node, digest string) bool
// Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads,
// read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no
// approved release is known to carry it. Nil: none is read.
Release func(fp string) (string, bool)
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
Released bool
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
@@ -77,9 +56,6 @@ type Module struct {
Now time.Time
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
Puts map[string]delivery.Put
// Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR
// 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none.
Needs []string
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
@@ -101,9 +77,6 @@ type Nodes struct {
Haves map[string]map[string]delivery.Have
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
Providers map[string]map[string][]string
// Reads is, by node, the reader capabilities its node-engine states in its node record (delivery.NodeRecord.ManifestReads,
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
Reads map[string][]string
}
// healthy says whether a have runs fp healthy.
@@ -157,36 +130,15 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
short(m.FP))
}
if words, lacking := m.readerLacking(node, n); lacking {
return false, delivery.RuleComposed, words
}
if _, ok := m.composition(node, m.FP); !ok {
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
}
if words, waiting := m.provisionWaiting(node); waiting {
return false, delivery.RuleProvisions, words
}
if words, waiting := m.verbWaiting(node); waiting {
return false, delivery.RuleProvisions, words
}
if words, waiting := m.busStateWaiting(node); waiting {
return false, delivery.RuleProvisions, words
}
return true, 0, ""
}
// readerLacking says why the module's want on node is held for the node's engine (ADR 0614 §4): a reader capability its
// manifest needs that the engine does not state, read as the baseline when it states none. It is held under rule 7 (the
// composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the
// engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states.
func (m Module) readerLacking(node string, n Nodes) (string, bool) {
missing := catalogue.MissingManifestReads(m.Needs, n.Reads[node])
if len(missing) == 0 {
return "", false
}
return graph.ReaderWaitWords(node, missing, m.Name, short(m.FP)), true // the words the delivery view names the edge by
}
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
func (m Module) provisionWaiting(node string) (string, bool) {
@@ -197,36 +149,8 @@ func (m Module) provisionWaiting(node string) (string, bool) {
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
}
keyed := m.Provisions(node)
// Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge,
// asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. A requirement catalogue.Awaited leaves out is
// not waited on when something offers it (a module's own name, a module the mesh provides itself, a provision the
// composition does not key: optional, a toolchain on the module's own machine, a login the controller makes). One that
// nothing offers is waited on (ADR 0603 §5: an unknown provider has provided nothing), so a consumer never moves ahead of
// a provider whose module the indexer has not yet taken in; it holds loudly, since the watchman raises the hold past its
// bound (ADR 0603 §8).
needed := m.Manifest.Requires
if m.Answering != nil {
needed = catalogue.Awaited(m.Manifest, m.Answering)
have := map[string]bool{}
for _, p := range keyed {
have[p.Provision] = true
}
for _, r := range slices.Sorted(slices.Values(m.Manifest.Requires)) {
if have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(needed, r) {
needed = append(needed, r) // the provisioner keys it from the manifests, whoever is placed yet
continue
}
if m.Offered == nil || m.Manifest.OptionalRequires[r] || len(m.Answering(r)) > 0 || m.Offered(r) || slices.Contains(needed, r) {
continue
}
needed = append(needed, r)
if !have[r] {
return fmt.Sprintf("no module offers %s yet for %s on %s: an unknown provider has provided nothing, so the want waits for one, and the watchman raises it past its bound", r, m.Name, node), true
}
}
}
for _, key := range slices.Sorted(maps.Keys(keyed)) {
if p := keyed[key]; p.State != provisions.Provided && slices.Contains(needed, p.Provision) {
if p := keyed[key]; p.State != provisions.Provided {
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
if p.Why != "" {
words += ": " + p.Why
@@ -344,18 +268,29 @@ type RecordNeed struct {
Words string
}
// Holding is a rule holding a node back, and why. The bus module is held back by the rules alone, as every module (ADR 0601 §1,
// §2): no bar of its own.
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
// running an old-form build, got no want and the pass said nothing).
type Holding struct {
Rule int
Words string
Bus bool
}
// Reason is the holding in words: the rule and its words.
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
func (h Holding) Reason() string {
if h.Bus {
return h.Words
}
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
}
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
func BusBarWords(node, prev, fp string) string {
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
}
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
@@ -490,6 +425,14 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
if need != nil {
out.Record = append(out.Record, *need)
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
out.Rules[node] = h
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
out.Neither[node] = h
}
}
continue
}
@@ -716,10 +659,10 @@ func short(fp string) string {
return part[:12]
}
// mustAsk says whether the move to the composition d on node waits for the operator's yes (need names the ask, ADR 0539 §14). A
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
// streams before it applies (§2); the assigner asks no snapshot for it.
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
// same fingerprint judged neither (a user list only, §7).
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
return nil, false
@@ -731,6 +674,9 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
if _, barred := m.busBar(before, had, running); barred {
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
}
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
return nil, false
}
@@ -740,7 +686,17 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
}
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
// readable want the one its have names (ADR 0539 §15 bars a new one).
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
prev = before.FP
if !had {
prev = running
}
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
}
// recorded says whether the module is of policy record: a person's choice at the assigner over everything but the bus (ADR
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
func (m Module) recorded(d delivery.ComposedDigest) bool {
@@ -9,8 +9,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/provisions"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
@@ -106,111 +104,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
}
}
// ADR 0614 §4: a want is held on a node whose engine does not state a reader capability the manifest needs, under rule 7
// with the capability as its words, and moves once the engine states it; an engine stating nothing reads the baseline,
// so a manifest of the baseline's words, the node-engine's own new build among them, is not held.
hostNeeds, err := catalogue.ManifestNeeds([]byte(`{"module":"mesh-host","version":"1","slug":"host","build":{"artifacts":[` +
`{"name":"host-arch","kind":"bundle","language":"go","system":"arch","from":"cmd/mesh-host"}]},"consumes":["restic.backed-up"],` +
`"state":[{"name":"health"}],"config":{"x":{"type":"number","minimum":1,"default":50,"runtime":true}},` +
`"resources":[{"id":"k","type":"file","path":"/var/lib/k","content":"${config:x}\n"}]}`))
if err != nil {
t.Fatal(err)
}
for _, c := range []struct {
why string
name string
needs []string
reads map[string][]string
held bool
}{
{"a word the baseline lacks, on engines stating nothing", "mail", []string{"word:module", "word:tool-runner"}, nil, true},
{"the same, once both engines state it", "mail", []string{"word:module", "word:tool-runner"},
map[string][]string{"ace": catalogue.ManifestReads(), "g14": catalogue.ManifestReads()}, false},
{"the baseline's words on engines stating nothing", "mail", []string{"word:claims.serves", "seat:mesh-catalog", "form:config"}, nil, false},
{"the node-engine's new build, whose contract has more words than its manifest uses", "mesh-host", hostNeeds, nil, false},
} {
m, n := ready()
m.Name, m.Needs, n.Reads = c.name, c.needs, c.reads
n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"]
out := Wants(m, n, wantedOld(), nil)
if moved := out.Wants["g14"].FP == fNew; moved == c.held {
t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"])
}
if r := out.Rules["g14"]; c.held && (r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, "word:tool-runner") ||
!strings.Contains(r.Words, "node-engine of g14")) {
t.Errorf("%s: held by %+v, want rule 7 naming the capability", c.why, r)
}
if w := out.Wants["g14"]; c.held && (w.Rule != delivery.RuleComposed || w.Newest != fNew) {
t.Errorf("%s: g14's want %+v, want it kept at what it runs and naming rule 7", c.why, w)
}
}
// ADR 0614 §2, §3: a want waits, under the provisions' rule, until each verb it invokes is served by a running build,
// and moves once one is; a verb a running build serves already adds no wait, and an edge the graph leaves out on a cycle
// is not among Invokes, so it holds nothing; nothing read of what runs waits for none.
for _, c := range []struct {
name string
invokes []string
served []string
nilRead bool
holds string
}{
{"a verb no running build serves", []string{"store.store_put", "seat:mesh-catalog.modules"}, []string{"store.store_put"}, false, "seat:mesh-catalog.modules"},
{"once a build serving it runs", []string{"seat:mesh-catalog.modules"}, []string{"seat:mesh-catalog.modules"}, false, ""},
{"an edge left out on a cycle", nil, nil, false, ""},
{"nothing read of what runs", []string{"seat:mesh-catalog.modules"}, nil, true, ""},
// ADR 0614 §5: a module asking bus state is held while no build offering bus-state runs, and moves once one does.
{"bus state no running build offers", nil, nil, false, graph.BusState},
{"bus state a running build offers", nil, []string{graph.BusState}, false, ""},
{"bus state its own module offers", nil, nil, false, ""},
} {
m, n := ready()
m.Invokes = c.invokes
if !c.nilRead {
m.Served = func(v string) bool { return slices.Contains(c.served, v) }
}
if strings.HasPrefix(c.name, "bus state") {
m.Manifest.BusState = []catalogue.BusStateEntry{{Name: "cache", Storage: "key-value"}}
m.Offering = func(p string) bool { return slices.Contains(c.served, p) }
if c.name == "bus state its own module offers" {
m.Manifest.Provides = []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}
}
}
words := graph.VerbWaitWords(c.holds, "mail", "g14")
if c.holds == graph.BusState {
words = graph.BusStateWaitWords("mail", "g14")
}
out := Wants(m, n, wantedOld(), nil)
if c.holds == "" {
if out.Wants["g14"].FP != fNew {
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
}
continue
}
if out.Wants["g14"].FP == fNew || out.Rules["g14"].Rule != delivery.RuleProvisions || out.Rules["g14"].Words != words {
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
}
}
// ADR 0603 §9: the newest source revision governs. The module now at fNew no longer requires the database an older revision
// asked for and requires oidc instead; the seat's keys follow the newest manifest, so the older edge is no wait: a provided
// oidc moves the want, an asked one holds it and names oidc and never database.
for _, c := range []struct {
name string
state, older string
moves bool
}{{"the replaced edge provided, the older edge retiring", provisions.Provided, provisions.Retiring, true},
{"the replaced edge provided, the older edge asked", provisions.Provided, provisions.Asked, true},
{"the replaced edge asked, the older edge retiring", provisions.Asked, provisions.Retiring, false}} {
m, n := ready()
m.Manifest.Requires = []string{"oidc"}
m.Provisions = func(node string) map[string]provisions.Provision {
return map[string]provisions.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state},
node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}}
}
out := Wants(m, n, wantedOld(), nil)
if got := out.Wants["g14"].FP == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) {
t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words)
}
}
// 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named; and only one the
// composition keys (catalogue.Awaited, issue 774), replaying 2026-10-30: grafana (not handed) waited on ace for
// oidc-client, which the controller still makes, and mesh-build-go (handed) for go-toolchain on its own machine.
@@ -262,24 +155,6 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
}
}
}
// ADR 0603 §5: a requirement nobody offers is an unknown provider and waits, loudly; one something offers (a module's own
// name, a module the mesh provides itself) is left to catalogue.Awaited as before and does not.
for _, c := range []struct {
name, requires string
offered bool
}{{"a module's own name (node-tools)", "node-tools", true}, {"a module the mesh provides itself (mesh-addressing)", "mesh-addressing", true},
{"a provision nobody offers", "database", false}} {
m, n := ready()
m.Manifest.Requires = []string{c.requires}
m.Answering = func(string) []catalogue.Manifest { return nil }
m.Offered = func(p string) bool { return c.offered && p == c.requires }
m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} }
out := Wants(m, n, wantedOld(), nil)
if got := out.Wants["g14"].FP == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != delivery.RuleProvisions ||
!strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) {
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
}
}
// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
// healthy held photos, and novox offline left a module with no canary.
@@ -764,13 +639,11 @@ func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T)
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
// ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node
// backs its streams up before it applies.
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 || len(out.Neither) != 0 {
t.Errorf("the bus did not move by the rules: %+v", out)
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew {
t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"])
}
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}}
out := Wants(m, n, running, nil)
@@ -823,6 +696,23 @@ func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
}
m.Puts = nil
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
// condition stays raised. A node with a live want keeps it and is not on neither list.
m.Composed = composedEverywhere
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
if w, wrote := out.Wants["ace"]; wrote {
t.Errorf("the bus bar wrote ace's want %+v", w)
}
h, on := out.Neither["ace"]
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
!strings.Contains(h.Words, "assigner_put") {
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
}
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
}
stepsOfAPut(t)
}
@@ -999,8 +889,8 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
}
}
// A node that runs the bus with no readable want gets its want by the rules, at the fingerprint it runs or at a new one alike: the
// bus has no bar of its own (ADR 0601 §1).
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred.
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
m, n, _ := movingTo(delivery.JudgedNeither)
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
@@ -1009,8 +899,8 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
}
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok)
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok {
t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w)
}
}
@@ -34,9 +34,8 @@ func TestManifest(t *testing.T) {
reads := slices.Clone(m.Reads)
slices.Sort(reads)
// ADR 0593 §8: it composes nothing, so it reads no candidate, held or resolved value, compose step or build log; it reads
// every node's published compositions to compare them, and the gate's source and merges, for the release that carries a build (ADR 0601 §1);
// and the indexer's module graph, whose verb edges a want waits on (ADR 0614 §2, §3).
want := []string{"mesh-delivery-builder.builds", "mesh-delivery-gate.merges", "mesh-delivery-gate.source", "mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
// every node's published compositions to compare them.
want := []string{"mesh-delivery-builder.builds", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
"mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed",
"mesh-provisioner.provisions", "node-engine.have"}
if !reflect.DeepEqual(reads, want) {
@@ -25,9 +25,6 @@
],
"reads": [
"mesh-delivery-builder.builds",
"mesh-delivery-gate.merges",
"mesh-delivery-gate.source",
"mesh-delivery-indexer.graph",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-teller.answers",
@@ -3,7 +3,7 @@ module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
github.com/nats-io/nats.go v1.54.0
)
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -98,7 +98,7 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " +
"log held more than one answer carries: ask again from the last build's time. What the operator's command nox builds reads. Read-only.",
"log held more than one answer carries: ask again from the last build's time. What mesh-catalog registers a build from. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
Run: func(args map[string]any) (any, error) {
@@ -131,7 +131,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
trunks.start()
b.Trunk = trunks.head
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
// indexer and the assigner through their reads, and by mesh-catalog through the tool builder_builds_since.
b.Log = func(e delivery.BuildLogEntry) {
if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil {
logger.Printf("%v", err)
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2 -2
View File
@@ -1,7 +1,7 @@
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -944,7 +944,7 @@ func LogStarted(module string, m delivery.Module, ref, by string, at time.Time)
}
// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it
// was built against and each artifact made, as the done key has them: what the operator's command nox builds shows of a build.
// was built against and each artifact made, as the done key has them: what mesh-catalog registers a build from.
func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry {
e := LogStarted(module, m, ref, by, at)
e.Event, e.Outcome, e.Manifest = delivery.BuildEndedEvent, result.State, m.Manifest
@@ -565,7 +565,6 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old))
r.b.Wait()
if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
t.Run("a recipe that moves builds the module again under its new fingerprint", aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint)
t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls))
}
// A recipe that differs is a different toolchain too, and a value that names none (written before) is not built.
@@ -590,45 +589,6 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt
}
}
// ADR 0603 §3 (D9): a source revision changing the Go build module's recipe and a Go module builds the module at once with
// the recipe the nodes run and again, a second key under the new fingerprint, once the agreement moves to the new recipe;
// the first key is neither rebuilt nor waited on.
func aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint(t *testing.T) {
r := newRig(t, goBundle)
r.consider(t, "modules/m")
calls := len(r.bm.calls)
fp2 := "src2:" + hex64('2')
for _, n := range []string{"ace", "novox"} {
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "2", At: time.Now()})
if err != nil {
t.Fatal(err)
}
r.tools.Put(buildrun.GoBuildModule, n+".go", b, false)
}
r.b.Reconsider(context.Background())
r.b.Wait()
if len(r.bm.calls) != calls {
t.Errorf("%d calls, was %d: the first fingerprint was built again for the new recipe", len(r.bm.calls), calls)
}
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp2,
Manifest: mdigest('a'), Path: "modules/m", Toolchains: map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "2"}}})
if err != nil {
t.Fatal(err)
}
r.b.Module(context.Background(), "m", data)
r.b.Wait()
second, err := delivery.BuilderBuildKey("m", fp2)
if err != nil {
t.Fatal(err)
}
if v := valueOf(t, r.st, second); v.State != delivery.BuildDone || v.Toolchains["go"].Recipe != "2" {
t.Errorf("second build = %+v, want done with recipe 2", v)
}
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
t.Errorf("first build = %+v, want it left done", v)
}
}
func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4
r := newRig(t, goBundle)
r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false)
@@ -103,25 +103,15 @@ func TryTools(tr Trying) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
return []stdio.Tool{
{Name: check.TryTool,
Description: "Try a delivery group, or one pushed head, before its pull requests are opened (try a group, group try, try several repositories): " +
"name the members, each a repository and the exact commit asked (never the branch's head now, so a push after the ask changes nothing of what is tried), " +
"or one repository and its head for a try of one member (the seat verb try of mesh-delivery-checker takes members from the registry milestone on; until it is delivered, ask a group through this tool). The checker judges it by the code a group's pull-request check runs, so the two cannot drift: " +
"each member merged onto its trunk's head, the unit and the integration level of every member through the build modules, the manifest validator over " +
"the members' manifests, the module graph, and the composition on every placed node. Every failure is named (each member, level, test, manifest and node), not " +
"the first. A try owes the forge nothing (no status, no Approve asked) and writes no verdict of a pull request; what its passing levels prove is kept, so a pull request's " +
"check of the same heads on the same trunk runs only what the try did not pass. A try narrowed to a level or to some tests judges only those, no manifest and no composition, and keeps nothing of tests. " +
"Answers the try's id at once; " + check.TryReadTool + " reads where it stands, " + check.TryEndTool + " ends it by hand. A try of the same commits that waits or runs is the answer to a second ask " +
"(whatever the order of the members); a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide. A repository the checker checks no level of (one whose trunk the gate does not name) " +
"is refused. A try whose ask no build node took within the step bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
Input: map[string]any{"type": "object", "properties": map[string]any{
"members": map[string]any{"type": "array", "description": "the group: one repository at most once, each at the exact commit asked and the trunk commit it is merged onto",
"items": map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"}}}},
"repository": map[string]any{"type": "string", "description": "<owner>/<name>, for a try of one member (instead of members)"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits (with repository)"},
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent (with repository)"},
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"},
"tests": map[string]any{"type": "string", "description": "a go test -run pattern: only those tests run, and nothing is kept"},
"level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}, "description": "only this level; both when absent"}}},
Run: func(args map[string]any) (any, error) {
@@ -136,32 +126,16 @@ func TryTools(tr Trying) []stdio.Tool {
if tr.Now != nil {
now = tr.Now
}
var members []check.TryAskMember
if raw, ok := args["members"]; ok {
list, isList := raw.([]any)
if !isList {
return nil, fmt.Errorf("members is a list of {repository, head, trunk}")
}
for _, item := range list {
m, isMap := item.(map[string]any)
if !isMap {
return nil, fmt.Errorf("a member is {repository, head, trunk}")
}
members = append(members, check.TryAskMember{Repository: str(m, "repository"), Head: str(m, "head"), Trunk: str(m, "trunk")})
}
}
id, err := check.AskTry(tr.Tries, trunks, check.TryAsk{Repository: str(args, "repository"), Head: str(args, "head"),
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level"), Members: members}, now())
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level")}, now())
if err != nil {
return nil, err
}
return map[string]any{"id": id}, nil
}},
{Name: check.TryReadTool,
Description: "Where a try (of a group or of one head) stands, read whole, as stored: queued, running, or its end (pass, fail, or error when it could not run, or " +
"when its manifests and composition could not be judged), with what was said of it, every member's commit and the trunk commit it was merged onto, the time it took, " +
"every failure one line each (members, levels, tests, manifests, the composition per node), the levels' tests run and kept, the composition on every placed node, " +
"the notes of the check (manifests judged, module graph, kept verdicts); while it runs, each ask it put as soon as it is put, " +
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
@@ -172,7 +146,7 @@ func TryTools(tr Trying) []stdio.Tool {
return check.ReadTry(tr.Tries, str(args, "id"))
}},
{Name: check.TryEndTool,
Description: "End a try of a group (or of one head) that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
"takes it within the step bound. A try that ended already is refused.",
@@ -196,24 +170,6 @@ func TryTools(tr Trying) []stdio.Tool {
}
}
// OpenTriesTools is the tool that lists the tries that wait or run, each with its members at their commits, and whether it is
// a whole try (the indexer writes the candidate of a whole try's group): the read a part uses at its start to find the tries
// whose start event it missed, instead of reading the checker's state.
func OpenTriesTools(tr Trying) []stdio.Tool {
return []stdio.Tool{{Name: check.TriesTool,
Description: "The tries that wait or run now (a try of a group or of one head; try.started and try.ended on the checker's stream say the same), each with its id, state, its members at their exact commits and " +
"trunk commits, and whether it is a whole try (not narrowed to a level or to tests, so it is composed and its manifests judged). For a part that " +
"missed the checker's try.started or try.ended event (the indexer at its start); checker_try_read reads one try whole. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
if tr.Tries == nil {
return nil, fmt.Errorf("the tries state cannot be read here")
}
open, err := check.OpenTries(tr.Tries)
return map[string]any{"tries": open}, err
}}}
}
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
const Seat = "mesh-delivery-checker"
@@ -288,9 +244,7 @@ func Tools(r check.Reader) []stdio.Tool {
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
"deaf, a try or check whose ask is given up waits until its bound. builds_not_ended names, of each running check, the builder's builds of " +
"its candidate's changed modules that are neither done nor failed: the check waits on them before it composes, within its build-wait-bound " +
"(novox/hq ADR 0603 §10); a failed one fails the check: rebuild it with the builder's builder_build_again, and the check asks itself again when the build is done. Read-only.",
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) { return r.Status() }},
}
@@ -329,7 +283,7 @@ func main() {
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)),
AskState: read(stdio.State(check.AsksState)), BuildState: read(stdio.State("mesh-delivery-builder.builds"))}
AskState: read(stdio.State(check.AsksState))}
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
tries := stdio.State(check.TriesState)
source := stdio.State("mesh-delivery-gate.source")
@@ -369,7 +323,7 @@ func main() {
_, err := verdicts.Update(key, json.RawMessage(value), revision)
return err
}}
tools := append(append(append(append(Tools(r), TryTools(trying)...), OpenTriesTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
if err := stdio.Serve("", logged(tools)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
@@ -8,8 +8,6 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"mesh-delivery-checker/internal/check"
)
@@ -108,39 +106,6 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
t.Error("the verb tried a repository nothing checks")
}
// A group try (issue 1921): the members, each an exact commit; members and one repository together are refused; the
// description says what it does in the words a search for "try a group" finds.
later := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(2, 5) },
Trunks: func() (map[string]string, error) {
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
}})
g, err := later[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}}})
if err != nil || g.(map[string]any)["id"] == id {
t.Errorf("a group try = %v, %v, want a try of its own", g, err)
}
if _, err := tools[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}},
"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40)}); err == nil {
t.Error("members and a repository were both taken")
}
if _, err := tools[0].Run(map[string]any{"members": "novox/mesh-delivery"}); err == nil {
t.Error("members that are no list were taken")
}
for _, tl := range tools {
for _, words := range []string{"group"} {
if !strings.Contains(strings.ToLower(tl.Description), words) {
t.Errorf("%s says nothing of a %s: %q", tl.Name, words, tl.Description)
}
}
}
if d := strings.ToLower(tools[0].Description); !strings.Contains(d, "try a group") || !strings.Contains(d, "exact commit") || !strings.Contains(d, "every failure") ||
!strings.Contains(d, "composition on every placed node") || !strings.Contains(d, "owes the forge nothing") || !strings.Contains(d, "kept") {
t.Errorf("the try's description does not say what a group try does: %q", d)
}
if open := OpenTriesTools(Trying{Tries: store}); len(open) != 1 || open[0].Name != check.TriesTool || !strings.Contains(open[0].Description, "try.started") {
t.Errorf("open tries tool = %+v", open)
} else if out, err := open[0].Run(nil); err != nil || len(out.(map[string]any)["tries"].([]tryevent.Open)) != 2 {
t.Errorf("open tries = %v, %v, want the two tries asked", out, err)
}
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
if err != nil || ended.(map[string]any)["state"] != "error" {
@@ -398,7 +398,7 @@ func startBed(t *testing.T) *bed {
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
delivery.TurnsBucket: delivery.TurnsHistory, delivery.BuilderBuildsBucket: 1} {
delivery.TurnsBucket: delivery.TurnsHistory} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
if err != nil {
t.Fatal(err)
@@ -578,72 +578,20 @@ func (b *bed) owed(t *testing.T, number int, head, state string) delivery.Owed {
// TestCheckerJudgesACandidateAndOwesItsStatus: a candidate the indexer wrote, one pull request of mesh-delivery, is fetched
// merged onto the trunk the gate names, digested and tested at both levels on the node, kept, previewed; the verdict passes
// with the counts of both levels and the composition not run, and the status mesh/check is owed on the head as success.
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1, 1929/AC1, ADR 0603 §10
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1, 1929/AC1
b := startBed(t)
// ADR 0603 §10: the candidate changes the module m, whose manifest the trunk holds; the builder has not built m's
// candidate fingerprint yet, so the check waits on the builder's key before it composes.
manifest := `{"module":"m"}`
digest := delivery.ManifestDigest([]byte(manifest))
stored, err := delivery.EncodeManifest(delivery.Manifest{Module: "m", JSON: manifest})
if err != nil {
t.Fatal(err)
}
mkey, _ := delivery.ManifestsKey(digest)
if _, err := b.buckets[delivery.ManifestsBucket].Put(b.ctx, mkey, stored); err != nil {
t.Fatal(err)
}
known, err := delivery.EncodeModule(delivery.Module{Repository: repo, Commit: trunk, Path: "modules/m", Manifest: digest, None2: "the fingerprint is computed with its chain"})
if err != nil {
t.Fatal(err)
}
if _, err := b.buckets[delivery.ModulesBucket].Put(b.ctx, "m", known); err != nil {
t.Fatal(err)
}
fp := "src2:" + strings.Repeat("f", 64)
p := pull(repo, 7, head1)
p.Modules, p.Fps = []string{"m"}, map[string]delivery.CandidateModule{"m": {FP2: fp, Manifest: digest}}
key := b.candidate(t, "feat/x", p)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
// ADR 0593 §7, §9 (D12): n1 and n2 are placed; n1's engine answers compose-candidate on the bus, n2's never does.
asked := b.engine(t, "n1", "n2")
b.start(t, nil)
b.said(t, "waits on the builder's builds of m before it composes")
if got := asked(); len(got) != 0 {
t.Fatalf("the engines were asked %+v before m's build ended", got)
}
bkey, _ := delivery.BuilderBuildKey("m", fp)
building, _ := json.Marshal(delivery.BuilderBuild{V: 1, State: delivery.BuildBuilding})
if _, err := b.buckets[delivery.BuilderBuildsBucket].Put(b.ctx, bkey, building); err != nil {
t.Fatal(err)
}
if v := b.verdict(t, key, func(delivery.Verdict) bool { return true }); v.State != delivery.VerdictRunning || len(asked()) != 0 {
t.Fatalf("verdict %s and %d asks while m builds: the check composes only once its build ended", v.State, len(asked()))
}
// m's build fails: the check fails naming it. Built again (builder_build_again writes the key done), the check asks
// itself again for the same heads, with no recheck, and passes.
failedBuild, _ := json.Marshal(delivery.BuilderBuild{V: 1, State: delivery.BuildFailed, Error: "go build: undefined: x"})
if _, err := b.buckets[delivery.BuilderBuildsBucket].Put(b.ctx, bkey, failedBuild); err != nil {
t.Fatal(err)
}
if v := b.verdict(t, key, ended); v.State != delivery.VerdictFail || v.Failed.Test != check.BuildFailedTest+"m" {
t.Fatalf("verdict %+v, want it failed on m's build", v)
}
built, _ := json.Marshal(delivery.BuilderBuild{V: 1, State: delivery.BuildDone, Manifest: digest})
if _, err := b.buckets[delivery.BuilderBuildsBucket].Put(b.ctx, bkey, built); err != nil {
t.Fatal(err)
}
b.said(t, "the builds of m it failed on are done again, so it is checked again")
v := b.verdict(t, key, func(v delivery.Verdict) bool { return v.State == delivery.VerdictPass })
// The check asked again takes the packages the failed check ran from their kept verdicts.
if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Kept != 2 || v.Levels[delivery.LevelIntegration].Kept != 2 || v.Trunk[repo] != trunk {
v := b.verdict(t, key, ended)
if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 2 || v.Levels[delivery.LevelIntegration].Run != 2 || v.Trunk[repo] != trunk {
t.Fatalf("verdict = %+v", v)
}
if c := v.Composition; c == nil || c.State != delivery.CompositionPass || !slices.Equal(c.Machines, []string{"n1"}) ||
!strings.Contains(c.Words, "n2: "+delivery.ComposeNoAnswer) || len(v.Machines) != 1 || v.Machines[0] != "n1" {
t.Errorf("composition %+v machines %v", v.Composition, v.Machines)
}
if v.Composition != nil && strings.Contains(v.Composition.Words, "did not end") {
t.Errorf("the composition says %q: m's build ended within the bound", v.Composition.Words)
}
if got := asked(); len(got) == 0 || got[0].Candidate != key {
t.Errorf("n1's engine was asked %+v, want the check's candidate %s", got, key)
}
@@ -654,11 +602,10 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
t.Errorf("the check ask named the nodes %v, want [n1] from the have's node record", names)
}
o := b.owed(t, 7, head1, "success")
if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 0 run, 2 kept") {
if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run") {
t.Errorf("owed = %+v", o)
}
// Two checks of the same heads: the one failed on m's build and the one it asked itself again; the second runs no test.
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 2, "mesh-build-go.build_go_digest": 4, "mesh-build-go.build_go_test": 2} {
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_digest": 2, "mesh-build-go.build_go_test": 2} {
if got := b.fake.count(tool); got != want {
t.Errorf("%s called %d times, want %d", tool, got, want)
}
@@ -862,11 +809,9 @@ func (s triesStore) Create(key string, value []byte) error {
// TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed: a try written as checker_try writes it is taken by the running checker,
// judged through the build modules as a pull request's head merged onto its trunk, and ended pass in its key, with no
// verdict and no status owed (ADR 0501 §5). A whole try waits for the indexer's candidate of its group, which it reads on the
// key's watch, and writes the merge previews the indexer makes it from (issue 1921).
// verdict and no status owed (ADR 0501 §5).
func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
b := startBed(t)
asked := b.engine(t, "n1") // a whole try is composed on every placed node, as a group's check is
b.start(t, nil)
store := triesStore{b.buckets[check.TriesBucket], b.ctx}
id, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head1}, time.Now())
@@ -874,36 +819,19 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
t.Fatal(err)
}
var last check.Try
// The indexer's candidate of the try's group (here written by the test), at the heads of its members.
raw0, _ := store.Get(id)
first, _ := check.DecodeTry(raw0)
cand := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: check.TryCase(id, first, nil).HeadsHash,
Pulls: []delivery.CandidatePull{{Head: delivery.Head{Repository: repo, Head: head1}, Modules: []string{}}}}
cand.FPSet = delivery.FingerprintSet(cand.Pulls)
data, err := delivery.EncodeCandidate(cand)
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey(id)
if _, err := b.buckets[delivery.CandidatesBucket].Create(b.ctx, ckey, data); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
raw, _ := store.Get(id)
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
break
}
}
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" || last.Composition == nil {
t.Fatalf("try = %+v, want passed on the trunk with its composition made", last)
}
if got := asked(); len(got) != 1 || !strings.HasPrefix(got[0].Candidate, id+".") {
t.Errorf("the placed node's engine was asked %+v, want the try's candidate %s.<heads>", got, id)
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" {
t.Fatalf("try = %+v, want passed on the trunk", last)
}
if b.fake.count("mesh-build-go.build_go_fetch") == 0 {
t.Errorf("the try fetched nothing")
}
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket} {
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket, delivery.PreviewsBucket} {
if keys, _ := b.buckets[name].Keys(b.ctx); len(keys) != 0 {
t.Errorf("a try wrote %s: %v", name, keys)
}
@@ -8,17 +8,13 @@ import (
"sort"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"mesh-delivery-checker/internal/check"
)
@@ -29,19 +25,12 @@ type judgeDeps struct {
trunks *check.Trunks
kept check.KV
previews check.KV
// candidates answers the indexer's candidate of a try's group (try_candidates.go); nil leaves a whole try with no
// manifest or composition judged, said.
candidates check.TryCandidates
repos *repoModules
repos *repoModules
// published is the indexer's packages state as it is followed (packages.go).
published *publishedPackages
forge check.Forge
announce check.Announcer
builds check.BuildStates
// ends waits on the builder's builds of a candidate before the check composes (ADR 0603 §10); nil composes without waiting.
ends check.BuildEnds
// buildWait is the setting build-wait-bound; check.DefaultBuildWait when 0.
buildWait time.Duration
// compositions makes a check's composition from the engines' answers (ADR 0593 §7); nil leaves it not run.
compositions check.CompositionReader
// nodes names every node with a record; nil hands no names.
@@ -100,58 +89,6 @@ func (n nodeNames) Names(ctx context.Context) ([]string, error) {
return out, nil
}
// runningServes says what the running builds serve (novox/hq ADR 0614 §7), read when a check asks it: every module's have
// in the node-engine's have that runs healthy, the builder's build of its fingerprint, and the verbs its manifest serves
// (graph.Served). A build whose manifest cannot be read serves nothing here, said in the log.
type runningServes struct {
have *nodeNames
builds check.BuildStates
read check.ManifestReader
logf func(format string, args ...any)
}
func (r runningServes) Served(ctx context.Context) (func(string) bool, error) {
if r.have == nil || r.have.kv == nil || r.builds == nil || r.read == nil {
return nil, fmt.Errorf("the node-engine's have, the builder's builds or the indexer's manifests cannot be read")
}
lister, err := r.have.kv.ListKeys(ctx)
if err != nil {
return nil, err
}
haves := map[string]map[string]delivery.Have{}
for key := range lister.Keys() {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
continue
}
e, err := r.have.kv.Get(ctx, key)
if err != nil {
continue
}
if h, err := delivery.DecodeHave(e.Value()); err == nil {
if haves[node] == nil {
haves[node] = map[string]delivery.Have{}
}
haves[node][module] = h
}
}
served, unread := graph.Served(haves, func(module, fp string) ([]string, bool) {
b, found, err := r.builds.Build(ctx, module, fp)
if err != nil || !found || b.Manifest == "" {
return nil, false
}
m, err := r.read.Manifest(ctx, b.Manifest)
if err != nil {
return nil, false
}
return graph.ServesOf(m), true
}, nil)
if len(unread) > 0 && r.logf != nil {
r.logf("the manifest of the running build of %v cannot be read, so it serves nothing the check counts (ADR 0614 §7)", unread)
}
return served, nil
}
func newJudge(d judgeDeps) check.Judge {
var nodes interface {
Names(ctx context.Context) ([]string, error)
@@ -159,12 +96,8 @@ func newJudge(d judgeDeps) check.Judge {
if d.nodes != nil {
nodes = d.nodes
}
ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf}
if d.nodes != nil && d.builds != nil && d.repos != nil {
ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf}
}
return ex
return &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Predefined: d.predefined, Logf: d.logf}
}
// manifestKV reads a manifest by its digest.
@@ -189,42 +122,6 @@ type repoModules struct {
repoOf map[string]string // by module: its repository
commit map[string]string // by module: the commit its record was computed at (the commit the mesh runs)
typed map[string]string // by module: its manifest digest
parsed map[string]catalogue.Manifest // by manifest digest: the manifest as read, once (a digest's manifest never changes)
}
// Manifest is the kept manifest of a digest, read from the indexer's manifests state once and then from memory.
func (r *repoModules) Manifest(ctx context.Context, digest string) (catalogue.Manifest, error) {
r.mu.Lock()
m, known := r.parsed[digest]
r.mu.Unlock()
if known {
return m, nil
}
key, err := delivery.ManifestsKey(digest)
if err != nil {
return catalogue.Manifest{}, err
}
raw, _, found, err := r.manifests.Get(ctx, key)
if err != nil {
return catalogue.Manifest{}, err
}
if !found {
return catalogue.Manifest{}, fmt.Errorf("the indexer keeps no manifest %s", digest)
}
kept, err := delivery.DecodeManifest(raw)
if err != nil {
return catalogue.Manifest{}, err
}
if m, err = parseManifest(kept.JSON); err != nil {
return catalogue.Manifest{}, err
}
r.mu.Lock()
if r.parsed == nil {
r.parsed = map[string]catalogue.Manifest{}
}
r.parsed[digest] = m
r.mu.Unlock()
return m, nil
}
// Put takes a change of the indexer's modules bucket in: a module's repository and the languages its manifest names.
@@ -360,69 +257,6 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu
return b, err == nil, err
}
// buildEnds hears the builder's build keys of a candidate on the bus (mesh-delivery-builder_builds): a watch of exactly
// those keys gives each one's value now and every write the builder makes after, so the check waits on the part that owns
// the fact and polls nothing (novox/hq ADR 0603 §10).
type buildEnds struct{ kv jetstream.KeyValue }
func (e buildEnds) Ended(ctx context.Context, builds map[string]string) (map[string]delivery.BuilderBuild, error) {
return e.until(ctx, builds, func(b delivery.BuilderBuild) bool {
return b.State == delivery.BuildDone || b.State == delivery.BuildFailed
})
}
func (e buildEnds) Done(ctx context.Context, builds map[string]string) bool {
done, err := e.until(ctx, builds, func(b delivery.BuilderBuild) bool { return b.State == delivery.BuildDone })
return err == nil && len(done) == len(builds)
}
// until hears the builds' keys until each holds a build that ended holds, or ctx ends, and answers those that do, by module.
func (e buildEnds) until(ctx context.Context, builds map[string]string, ended func(delivery.BuilderBuild) bool) (map[string]delivery.BuilderBuild, error) {
byKey := map[string]string{}
keys := make([]string, 0, len(builds))
for module, fp2 := range builds {
key, err := delivery.BuilderBuildKey(module, fp2)
if err != nil {
return nil, err
}
byKey[key] = module
keys = append(keys, key)
}
out := map[string]delivery.BuilderBuild{}
if len(keys) == 0 {
return out, nil
}
w, err := e.kv.WatchFiltered(ctx, keys)
if err != nil {
return nil, err
}
defer func() { _ = w.Stop() }()
for len(out) < len(keys) {
select {
case <-ctx.Done():
return out, nil // the bound: the builds not ended are left out
case entry, open := <-w.Updates():
if !open {
return out, nil
}
if entry == nil || entry.Operation() != jetstream.KeyValuePut {
continue // the end of the values now, or a key deleted: it has not ended
}
module, ours := byKey[entry.Key()]
if !ours {
continue
}
b, err := delivery.DecodeBuilderBuild(entry.Value())
if err == nil && ended(b) {
out[module] = b
} else {
delete(out, module) // asked again (builder_build_again), or not a build the check can read: not ended
}
}
}
return out, nil
}
// ManifestDigests is the manifest digest of every module the indexer holds, by module: the catalogue the indexer's judge
// judges a change's manifests in (novox/hq ADR 0578 §2).
func (r *repoModules) ManifestDigests() map[string]string {
@@ -21,7 +21,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -167,19 +166,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
// says nothing of the builds, until the checker starts again.
var builds check.BuildStates
var ends check.BuildEnds
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds, and a check composes without waiting on them, until the checker starts again", logger.Printf); kv != nil {
builds, ends = buildReader{kvStore{kv}}, buildEnds{kv}
}
if ends != nil {
ch.Rebuilds = ends // a check failed on a build is asked again once that build is done again (ADR 0603 §10)
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil {
builds = buildReader{kvStore{kv}}
}
// The check's composition is the checker's, from what each placed node's engine answers (ADR 0593 §7, §9).
set, err := readSettings(getenv("MESH_CHECKER_SETTINGS"))
answerBound, err := composeAnswerBound(getenv("MESH_CHECKER_SETTINGS"))
if err != nil {
return err
}
answerBound := set.answer
holdCallsFor(answerBound)
verdicts := &manifestVerdicts{}
compositions := check.EngineCompositions{
@@ -190,13 +184,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Verdicts: verdicts.all,
}
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf,
// A whole try's start and end are said on the stream checks, for the indexer to write and delete the candidate of its group.
Say: func(ctx context.Context, event string, body tryevent.Event) error {
data, _ := json.Marshal(body)
_, err := js.Publish(ctx, eventSubject(event), data, jetstream.WithRetryAttempts(10), jetstream.WithRetryWait(time.Second))
return err
}}
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
announce := func(ctx context.Context, event, key string) {
body, _ := json.Marshal(map[string]string{"key": key})
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
@@ -209,7 +197,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The indexer's packages state (ADR 0593 §2): followed once the bus has it, so a Go check builds every mesh package at
// its published version; until then a check builds each at the version its go.mod names.
published := &publishedPackages{}
compositions.Published = published
process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "a Go check builds each mesh package at the version its go.mod names",
logger.Printf).Ready(func(kv jetstream.KeyValue) {
w, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) })
@@ -224,8 +211,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
ch.Judge = newJudge(judgeDeps{
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, candidates: tryCandidates{buckets[delivery.CandidatesBucket]},
repos: repos, published: published, builds: builds, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
repos: repos, published: published, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce, compositions: compositions,
nodes: openNodeNames(ctx, js, logger.Printf),
predefined: keptPredefined{process.AwaitBucket(ctx, js, conn, delivery.IndexerModule+"_"+delivery.IndexerPredefinedState,
@@ -341,9 +328,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
})
}()
each := func(e jetstream.KeyValueEntry) {
if check.IsTryGroup(e.Key()) {
return // a group try's candidate: its try judges it, and the forge is owed nothing (issue 1921)
}
if opOf(e) {
ch.Candidate(ctx, e.Key(), e.Value())
} else {
@@ -10,57 +10,38 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
)
// The checker's own settings (its config, written by the node-engine into ${dir:state}/settings.json, named to the
// process by MESH_CHECKER_SETTINGS): compose-answer-bound, the seconds a check waits for a node's engine to answer
// compose-candidate (novox/hq ADR 0593 §7, §9: 60 by default), and build-wait-bound, the seconds a check waits for the
// builder's builds of its candidate before it composes (ADR 0603 §10: 1800 by default). The checker owns both.
// compose-candidate (novox/hq ADR 0593 §7, §9: 60 by default; the checker owns it).
// settings are the checker's settings as read.
type settings struct {
// answer is compose-answer-bound; buildWait is build-wait-bound.
answer, buildWait time.Duration
}
// readSettings reads the settings file at path: each value's default where no file or no value is there; a value that is
// no whole number of seconds within its range is refused, naming it.
func readSettings(path string) (settings, error) {
out := settings{answer: delivery.ComposeAnswerBound, buildWait: check.DefaultBuildWait}
// composeAnswerBound reads compose-answer-bound from the settings file at path: delivery.ComposeAnswerBound where no
// file or no value is there; a value that is no number of seconds from 1 to 600 is refused, naming it.
func composeAnswerBound(path string) (time.Duration, error) {
if path == "" {
return out, nil
return delivery.ComposeAnswerBound, nil
}
raw, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
return out, nil
return delivery.ComposeAnswerBound, nil
}
if err != nil {
return settings{}, err
return 0, err
}
var s struct {
Answer *float64 `json:"compose-answer-bound"`
BuildWait *float64 `json:"build-wait-bound"`
Bound *float64 `json:"compose-answer-bound"`
}
if err := json.Unmarshal(raw, &s); err != nil {
return settings{}, fmt.Errorf("%s is no settings file: %w", path, err)
return 0, fmt.Errorf("%s is no settings file: %w", path, err)
}
for _, v := range []struct {
name string
value *float64
min, max float64
into *time.Duration
}{{"compose-answer-bound", s.Answer, 1, 600, &out.answer}, {"build-wait-bound", s.BuildWait, 60, 7200, &out.buildWait}} {
if v.value == nil {
continue
}
if *v.value < v.min || *v.value > v.max || *v.value != float64(int(*v.value)) {
return settings{}, fmt.Errorf("%s is %v: a whole number of seconds from %v to %v", v.name, *v.value, v.min, v.max)
}
*v.into = time.Duration(*v.value) * time.Second
if s.Bound == nil {
return delivery.ComposeAnswerBound, nil
}
return out, nil
if *s.Bound < 1 || *s.Bound > 600 || *s.Bound != float64(int(*s.Bound)) {
return 0, fmt.Errorf("compose-answer-bound is %v: a whole number of seconds from 1 to 600", *s.Bound)
}
return time.Duration(*s.Bound) * time.Second, nil
}
// holdCallsFor makes a bus call wait the compose-answer-bound (novox/hq ADR 0593 §7), not the SDK's CallTimeout of 30 s:
@@ -8,25 +8,19 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
)
// ADR 0593 §7: the answer bound is the checker's setting compose-answer-bound, 60 s when the file or the value is
// absent; a value that is no whole number of seconds from 1 to 600 stops the start, naming it. ADR 0603 §10: the wait for
// the candidate's builds is build-wait-bound, 30 minutes when absent, from 60 to 7200 seconds.
// absent; a value that is no whole number of seconds from 1 to 600 stops the start, naming it.
func TestTheAnswerBoundIsTheCheckersSetting(t *testing.T) {
dir := t.TempDir()
for _, row := range []struct {
body string
want time.Duration
err bool
buildWait time.Duration
}{{"", delivery.ComposeAnswerBound, false, check.DefaultBuildWait}, {`{}`, delivery.ComposeAnswerBound, false, check.DefaultBuildWait},
{`{"compose-answer-bound": 90}`, 90 * time.Second, false, check.DefaultBuildWait}, {`{"compose-answer-bound": 0}`, 0, true, 0},
{`{"compose-answer-bound": 1.5}`, 0, true, 0},
{`{"compose-answer-bound": 60, "build-wait-bound": 600}`, 60 * time.Second, false, 10 * time.Minute},
{`{"build-wait-bound": 30}`, 0, true, 0}} {
body string
want time.Duration
err bool
}{{"", delivery.ComposeAnswerBound, false}, {`{}`, delivery.ComposeAnswerBound, false},
{`{"compose-answer-bound": 90}`, 90 * time.Second, false}, {`{"compose-answer-bound": 0}`, 0, true},
{`{"compose-answer-bound": 1.5}`, 0, true}} {
path := filepath.Join(dir, "settings.json")
os.Remove(path)
if row.body != "" {
@@ -34,10 +28,9 @@ func TestTheAnswerBoundIsTheCheckersSetting(t *testing.T) {
t.Fatal(err)
}
}
set, err := readSettings(path)
got := set.answer
if (err != nil) != row.err || got != row.want || set.buildWait != row.buildWait {
t.Errorf("%q: %s, %s, %v", row.body, got, set.buildWait, err)
got, err := composeAnswerBound(path)
if (err != nil) != row.err || got != row.want {
t.Errorf("%q: %s, %v", row.body, got, err)
}
// Issue 1126: the SDK's call gives up at CallTimeout (30 s), so the checker's calls must wait the bound read
// (an answer at 31 s is heard under 60 s), and the bound never shortens the SDK's own.
@@ -1,72 +0,0 @@
package main
import (
"context"
"fmt"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// candidateWaitsOnPreview is the None2 of a module the indexer wrote from a pull message alone, before it had the merge
// preview of the head (the indexer's NoneNoPreview): its fingerprint comes with the preview, so a candidate that still says
// it is not yet the candidate of the heads previewed. It moves to mesh-sdk's delivery package with the next change there.
const candidateWaitsOnPreview = "no merge preview is computed"
// tryCandidates answers the indexer's candidate of a try's group (novox/hq ADR 0593 §7, issue 1921): the candidate the
// engines compose and the manifest validator judges, which the indexer makes of the try's previews. It waits for it on the
// key's own watch, with no timer between reads, and gives up at the bound.
type tryCandidates struct{ kv jetstream.KeyValue }
func (t tryCandidates) TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error) {
key, err := delivery.CandidatesKey(group)
if err != nil {
return delivery.Candidate{}, err
}
if bound <= 0 {
bound = time.Minute
}
waiting, cancel := context.WithTimeout(ctx, bound)
defer cancel()
w, err := t.kv.Watch(waiting, key) // the value now, then each change
if err != nil {
return delivery.Candidate{}, fmt.Errorf("the indexer's candidate %s cannot be watched: %w", key, err)
}
defer func() { _ = w.Stop() }()
last := "the indexer holds none"
for {
select {
case <-waiting.Done():
return delivery.Candidate{}, fmt.Errorf("the indexer wrote no candidate of %s at heads %s within %s (%s)", group, heads, bound, last)
case e := <-w.Updates():
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
c, err := delivery.DecodeCandidate(e.Value())
switch {
case err != nil:
last = "what it holds is not a candidate: " + err.Error()
case c.HeadsHash != heads:
last = "it holds heads " + c.HeadsHash
case waitsOnPreview(c):
last = "it has not yet the merge previews' fingerprints"
default:
return c, nil
}
}
}
}
// waitsOnPreview says a module of the candidate has no fingerprint yet because its preview was not read.
func waitsOnPreview(c delivery.Candidate) bool {
for _, p := range c.Pulls {
for _, m := range p.Fps {
if m.None2 == candidateWaitsOnPreview {
return true
}
}
}
return false
}
@@ -1,4 +1,3 @@
{
"compose-answer-bound": 60,
"build-wait-bound": 1800
"compose-answer-bound": 60
}
+1 -5
View File
@@ -3,7 +3,7 @@ module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
@@ -25,7 +25,3 @@ require (
)
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
+2 -2
View File
@@ -1,7 +1,7 @@
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -29,30 +29,24 @@ const (
nodeEngineModule = "mesh-host"
// networkingRepository holds route-proxy, whose manifest the node-engine's proofs of a composition read.
networkingRepository = "novox/mesh-networking"
// contractsRepository and deliveryRepository hold the seat definitions mesh-sdk's bootstrapped seats are compared with: its
// tests read both beside it (beside.Dir "mesh-contracts" and "mesh-delivery"), at the group's head for a member, else at the
// trunk (a group try lays them out the same way: besideFor serves the check and the try).
contractsRepository = "novox/mesh-contracts"
deliveryRepository = DeliveryRepository
)
// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside
// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog
// checks/deps). Which commit of each is besideCommit's rule. No repository's tests read the controller, which ended by its
// unassignment (novox/hq ADR 0590, issue 1126), and its repository is emptied and archived (ADR 0593 §11), so it has no
// siblings placed either. mesh-sdk reads mesh-contracts and mesh-delivery (its bootstrapped seats against their contracts files). The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
// siblings placed either. The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
// §7): mesh-build-go builds it for a compose ask. The node-engine's read mesh-networking since route-proxy's proofs of a
// composition moved into them (ADR 0593).
var besideRules = map[string][]string{
nodeEngineRepo: {catalogueRepository, SDKRepository, networkingRepository},
SDKRepository: {catalogueRepository, nodeEngineRepo, contractsRepository, deliveryRepository},
SDKRepository: {catalogueRepository, nodeEngineRepo},
catalogueRepository: {SDKRepository, nodeEngineRepo},
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
}
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the node-engine at the head of the group's pull
// request for it when the group holds one, else at the commit the mesh runs (the commit of the indexer's module mesh-host,
// its trunk head when none is known); any other sibling, the catalogue
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the node-engine at the commit the mesh runs
// (the commit of the indexer's module mesh-host, its trunk head when none is known); any other sibling, the catalogue
// included, at the head of the group's pull request for it when the group holds one, else at its trunk, since a group's
// check judges each member against the group's heads (ADR 0482 §2, ADR 0573 §4; issue 1126: a judge a group changed was
// judged against the catalogue's trunk). And first, the change's own repository at the trunk it is merged onto, so a test
@@ -69,12 +63,6 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
}
for _, repo := range besideRules[p.Repository] {
switch {
case repo == nodeEngineRepo && c.member(repo) != nil:
// A group's check judges each member against the group's heads (ADR 0573 §4, ADR 0482 §2), and the group's
// node-engine is what runs after its merge: it is placed at the member's head, as any member is. Neither atTrunk
// nor runs, so no move of the trunk or of the running commit puts the verdict out of date (besideAtTrunk).
member := c.member(repo)
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
case repo == nodeEngineRepo:
commit := ""
if j.Runs != nil {
@@ -1,142 +0,0 @@
package check
import (
"context"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
// What the check builds of a candidate before it composes (novox/hq ADR 0603 §10). The builder builds every fingerprint the
// indexer's candidate carries, as a key of its own (builds.<module>.<fingerprint>), with the build modules that run; the
// check adds no build path of its own and waits on those keys (awaitBuilds), so the node-engines compose the candidate's
// modules with their builds. It does not wait on two kinds of changed module, and names them in its composition's warnings:
//
// - one built by a build module the change alters (the recipe of mesh-build-go or mesh-build-node enters the fingerprint
// of every module it builds, ADR 0530 §1): its candidate build is the running build module's, and the check cannot run
// the candidate's build module to build it (it runs it only for its own digest and tests, ADR 0535 §1), so it is not
// composed with a build of the changed build module; it is built again once the agreed toolchain moves (ADR 0603 §3).
// The build module's own build is waited on, as any other: the running build module builds its new source.
// - one that stands on a module the change alters (build.on): its candidate fingerprint names the base's trunk
// fingerprint, and no build module builds on another module's artifact (ADR 0487 §1), so its build stands on the
// trunk's base; the base's own build is waited on.
//
// A module the trunk holds no record of yet is not built by the builder until it is merged, so it is not waited on either.
//
// A changed build module whose manifest invokes a seat verb or module tool no running build serves is one the check cannot
// run (ADR 0614 §7): the modules it builds are composed from their manifests, as above, and the words name the verb.
// buildsOfTheCheck are the builds the check waits on before it composes (module to fingerprint) and the words for the
// changed modules it composes without waiting on their build, one line each, sorted. A manifest that cannot be read leaves
// every build waited on and nothing said of build modules or bases (nothing is said that was not read).
func (st *state) buildsOfTheCheck(ctx context.Context) (map[string]string, []string) {
wait := map[string]string{}
changed := map[string]catalogue.Manifest{}
readable := true
for _, p := range st.c.Pulls {
for module, cm := range p.Fps {
if cm.FP2 == "" || cm.Manifest == "" {
continue // the builder builds no fingerprint of it (candidates.go: none of the new form, or no manifest)
}
wait[module] = cm.FP2
if !readable || st.j.Read == nil {
readable = false
continue
}
m, err := st.j.Read.Manifest(ctx, cm.Manifest)
if err != nil {
st.j.logf("%s: the manifest of %s cannot be read, so the modules built with a changed build module or base are not named and every build is waited on: %v", st.c.Key, module, err)
readable = false
continue
}
changed[module] = m
}
}
var words []string
if st.j.Runs != nil {
var unheld []string
for _, module := range slices.Sorted(maps.Keys(wait)) {
if st.j.Runs.CommitOf(module) == "" {
unheld = append(unheld, module)
delete(wait, module)
}
}
if len(unheld) > 0 {
words = append(words, fmt.Sprintf("%s: the trunk holds no record of it yet, so the builder builds it once it is merged and the check composes it without a build (ADR 0603 §10)",
strings.Join(unheld, ", ")))
}
}
if !readable {
return wait, words
}
for _, bm := range slices.Sorted(maps.Keys(buildrun.LanguageOf)) {
if _, alters := changed[bm]; !alters {
continue
}
var built []string
for module, m := range changed {
if _, isBuildModule := buildrun.LanguageOf[module]; !isBuildModule && buildsWith(m, buildrun.LanguageOf[bm]) {
built = append(built, module)
delete(wait, module)
}
}
slices.Sort(built)
if len(built) > 0 {
words = append(words, fmt.Sprintf("the change alters the build module %s: %s, which it builds, are built by the running %s, not the changed one, which the check cannot run, so the check does not wait on their builds; they are built again once the agreed toolchain moves (ADR 0603 §3, §10)",
bm, strings.Join(built, ", "), bm))
}
if unserved := st.unservedVerbs(ctx, changed[bm]); len(unserved) > 0 {
what := "nothing it builds is in the change"
if len(built) > 0 {
what = strings.Join(built, ", ") + ", which it builds, are composed from their manifests"
}
words = append(words, fmt.Sprintf("the changed build module %s invokes %s, which no running build serves, so the check cannot run it: %s (ADR 0614 §7)",
bm, strings.Join(unserved, ", "), what))
}
}
g := graph.OfManifests(changed)
for _, base := range slices.Sorted(maps.Keys(changed)) {
if on := g.Dependents(base); len(on) > 0 {
for _, module := range on {
delete(wait, module)
}
words = append(words, fmt.Sprintf("the change alters %s, a base of %s: their builds stand on the trunk's %s, since no build module builds on a candidate's base, so the check waits on the build of %s alone (ADR 0603 §10)",
base, strings.Join(on, ", "), base, base))
}
}
return wait, words
}
// buildsWith says whether a manifest has a bundle in the language: the artifacts whose fingerprint names that language's
// toolchain and its build module's recipe (an archive is packed with no toolchain, ADR 0478 clause 1).
func buildsWith(m catalogue.Manifest, language string) bool {
return m.Build != nil && slices.ContainsFunc(m.Build.Artifacts, func(a catalogue.Artifact) bool {
return a.Kind == catalogue.ArtifactBundle && a.Language == language
})
}
// unservedVerbs are the seat verbs and module tools m invokes (graph.InvokesOf) that no running build serves, sorted; none
// when nothing reads what runs, or what runs cannot be read (said in the log: nothing is said that was not read).
func (st *state) unservedVerbs(ctx context.Context, m catalogue.Manifest) []string {
if st.j.Serving == nil {
return nil
}
served, err := st.j.Serving.Served(ctx)
if err != nil {
st.j.logf("%s: what the running builds serve cannot be read, so no build module is named for a verb it invokes: %v", st.c.Key, err)
return nil
}
var out []string
for _, v := range graph.InvokesOf(m) {
if !served(v) {
out = append(out, v)
}
}
return out
}
@@ -2,12 +2,7 @@ package check
import (
"context"
"fmt"
"maps"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -69,102 +64,3 @@ func (j *Examiner) buildNotes(ctx context.Context, c *Case) []BuildNote {
}
return notes
}
// DefaultBuildWait is how long a check waits for the candidate's builds to end before it composes (the setting
// build-wait-bound, novox/hq ADR 0603 §10): past it the check composes without the builds not ended and names them.
const DefaultBuildWait = 30 * time.Minute
// BuildEnds waits on the builder's build keys (builds.<module>.<fingerprint>), hearing each key's changes on the bus as the
// builder writes them: the builder owns the fact, and nothing is polled.
type BuildEnds interface {
// Ended waits until the build of every module at its fingerprint (module to fingerprint) is done or failed, or ctx ends,
// and answers the builds that ended, by module. An error is only builds that cannot be heard at all.
Ended(ctx context.Context, builds map[string]string) (map[string]delivery.BuilderBuild, error)
// Done waits until the build of every module at its fingerprint is done (a failed one built again, builder_build_again),
// and says whether they all are; false when ctx ends first or they cannot be heard.
Done(ctx context.Context, builds map[string]string) bool
}
// BuildFailedTest is what a failure of the check names a failed build of the candidate by: the test `build: <module>`.
// A verdict failed on builds is asked again by the checker itself once each is built again (Checker.Rebuilt).
const BuildFailedTest = "build: "
// failedBuilds are the candidate's builds a failed verdict names as failed (BuildFailedTest), by module, at the fingerprint
// the case carries for it; none for a verdict that names no failed build.
func failedBuilds(v delivery.Verdict, cs *Case) map[string]string {
if v.State != delivery.VerdictFail {
return nil
}
out := map[string]string{}
for _, f := range append([]delivery.FailedFirst{v.Failed}, v.Failures...) {
module, named := strings.CutPrefix(f.Test, BuildFailedTest)
if !named {
continue
}
for _, p := range cs.Pulls {
if cm, ok := p.Fps[module]; ok && cm.FP2 != "" {
out[module] = cm.FP2
}
}
}
if len(out) == 0 {
return nil
}
return out
}
func (j *Examiner) buildWait() time.Duration {
if j.BuildWait > 0 {
return j.BuildWait
}
return DefaultBuildWait
}
// awaitBuilds waits for the candidate's builds before the check composes (ADR 0603 §10): the check builds what the builder
// builds of the candidate and composes with it. A failed build fails the check, naming the module and the builder's words;
// a build not ended within the bound is named, and the check composes without it. It answers the words of the composition's
// warnings: what the check composes without waiting on its build, and why.
func (st *state) awaitBuilds(ctx context.Context, v *delivery.Verdict) []string {
wait, words := st.buildsOfTheCheck(ctx)
if len(wait) == 0 {
return words
}
modules := slices.Sorted(maps.Keys(wait))
if v.State == delivery.VerdictFail {
// The check has failed already: its red status is never held back by a build (the lead, 2026-11-03).
return append(words, fmt.Sprintf("the check failed before its builds were waited on, so it composes at once, without waiting on the builds of %s (ADR 0603 §10)", strings.Join(modules, ", ")))
}
if st.j.BuildEnds == nil {
return append(words, fmt.Sprintf("the builds of %s cannot be heard, so the check composes without waiting on them (ADR 0603 §10)", strings.Join(modules, ", ")))
}
bound := st.j.buildWait()
waits := fmt.Sprintf("waits on the builder's builds of %s before it composes, at most %s (ADR 0603 §10)", strings.Join(modules, ", "), bound)
st.c.note(waits)
st.j.logf("%s %s", st.c.Key, waits)
at, cancel := context.WithTimeout(ctx, bound)
ended, err := st.j.BuildEnds.Ended(at, wait)
cancel()
if err != nil {
st.j.logf("%s: the builds of %s cannot be heard: %v", st.c.Key, strings.Join(modules, ", "), err)
return append(words, fmt.Sprintf("the builds of %s cannot be heard (%v), so the check composes without waiting on them (ADR 0603 §10)", strings.Join(modules, ", "), err))
}
var late []string
for _, module := range modules {
b, done := ended[module]
switch {
case !done:
late = append(late, module)
case b.State == delivery.BuildFailed:
f := delivery.FailedFirst{Test: BuildFailedTest + module, Words: fmt.Sprintf("the candidate's build at %s failed (rebuild with builder_build_again; the check asks itself again when the build is done): %s", wait[module], b.Error)}
if v.State != delivery.VerdictFail {
v.State, v.Failed = delivery.VerdictFail, f
continue
}
addFailure(v, f)
}
}
if len(late) > 0 && ctx.Err() == nil {
words = append(words, fmt.Sprintf("the builds of %s did not end within the build-wait-bound of %s, so the check composes without them (ADR 0603 §10)", strings.Join(late, ", "), bound))
}
return words
}
@@ -55,8 +55,6 @@ type Case struct {
Watch func(askID string, step *delivery.Step)
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
Progress *Progress
// Wait says what a whole try waits for now ("" when nothing); the try keeps it for try-read. Nil for a pull request's check.
Wait func(why string)
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
Kind string
@@ -4,8 +4,6 @@ import (
"context"
"errors"
"fmt"
"maps"
"slices"
"sort"
"strings"
"sync"
@@ -81,9 +79,6 @@ type Checker struct {
}
// Bound is how long one check may run; none when 0 (DefaultBound).
Bound time.Duration
// Rebuilds hears the builder's build keys a failed verdict names (failedBuilds): once each is written done, the check of
// those heads is asked again by the checker itself (ADR 0603 §10), as a failure of the nodes is; nil asks none again.
Rebuilds BuildEnds
// Asks keeps the asks each running check put, by verdict key (AsksState), for checker_ask_again; nil keeps none.
Asks AsksKV
// Link is the base of the address a status points at; the verdict key follows.
@@ -109,8 +104,6 @@ type Checker struct {
held map[string]string // by verdict key: the turn that holds its check off, as last said
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
settling map[string]*window // by candidates key: the window its heads settle in
rebuilt map[string]bool // by verdict key: every failed build its verdict names is done again
awaiting map[string]bool // by verdict key: its failed builds are heard (awaitRebuilt)
// outsideSaid are the candidates outside the mesh said in the log, by group and heads.
outsideSaid map[string]bool
}
@@ -169,7 +162,6 @@ func (ch *Checker) init() {
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{}
ch.rebuilt, ch.awaiting = map[string]bool{}, map[string]bool{}
ch.outsideSaid = map[string]bool{}
})
}
@@ -388,9 +380,8 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
// ADR 0536 §4). The checker holds no place and no capacity of its own.
ch.prepare(ctx, key, cs)
ch.prepare(ctx, cs)
ch.try(ctx, run, key, cs)
ch.hearFailedBuilds(ctx, key, cs) // a check it failed on a build is asked again once that build is done again
ch.mu.Lock()
more := ch.again[vkey]
delete(ch.again, vkey)
@@ -527,7 +518,7 @@ const (
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing
// to do.
func (ch *Checker) prepare(ctx context.Context, group string, cs *Case) {
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
if err != nil || !found {
@@ -540,10 +531,6 @@ func (ch *Checker) prepare(ctx context.Context, group string, cs *Case) {
case IsNodesFailure(v):
cs.Kind = KindNodes
return
case failedBuilds(v, cs) != nil:
cs.Kind = KindNodes // asked again on check once its builds are done again (awaitRebuilt), never by a recheck
ch.awaitRebuilt(ctx, group, cs.Key, failedBuilds(v, cs))
return
case IsConflictFailure(v):
return // a head git refused: its next check is whole (ADR 0548 §7)
case IsEnded(v), IsUntrunked(v):
@@ -700,8 +687,6 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
return nil, nil
case IsEnded(cur), ch.stale(cur, cs):
retake = true // heads held again after their check was ended unjudged (issue 1890), or a trunk that moved
case cur.State == delivery.VerdictFail && ch.takeRebuilt(cs.Key):
retake = true // every failed build it names is done again (ADR 0603 §10)
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
ch.mu.Lock()
at, ok := ch.nodesFail[cs.Key]
@@ -1085,61 +1070,3 @@ func (ch *Checker) movedSince(cs *Case) bool {
}
return false
}
// awaitRebuilt hears, once per verdict key, the builder's keys of the builds a failed verdict names: when each is written
// done again (builder_build_again), the check of those heads is asked again (claim retakes it), as a failure of the nodes
// is after the nodes changed, so no recheck by hand is needed (ADR 0603 §10). Heard on the builder's own writes, never
// polled; it ends with the checker.
func (ch *Checker) awaitRebuilt(ctx context.Context, group, vkey string, builds map[string]string) {
if ch.Rebuilds == nil {
return
}
ch.mu.Lock()
if ch.awaiting[vkey] {
ch.mu.Unlock()
return
}
ch.awaiting[vkey] = true
ch.mu.Unlock()
ch.wg.Add(1)
go func() {
defer ch.wg.Done()
done := ch.Rebuilds.Done(ctx, builds)
ch.mu.Lock()
delete(ch.awaiting, vkey)
if done {
ch.rebuilt[vkey] = true
}
c, known := ch.known[group]
ch.mu.Unlock()
if done && known && ctx.Err() == nil {
ch.logf("%s: the builds of %s it failed on are done again, so it is checked again (ADR 0603 §10)", vkey, strings.Join(slices.Sorted(maps.Keys(builds)), ", "))
ch.consider(ctx, group, c)
}
}()
}
// hearFailedBuilds starts awaitRebuilt for the case's verdict when it names failed builds (failedBuilds).
func (ch *Checker) hearFailedBuilds(ctx context.Context, group string, cs *Case) {
if ch.Rebuilds == nil || ctx.Err() != nil {
return
}
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
if err != nil || !found {
return
}
if v, err := delivery.DecodeVerdict(value); err == nil {
if builds := failedBuilds(v, cs); builds != nil {
ch.awaitRebuilt(ctx, group, cs.Key, builds)
}
}
}
// takeRebuilt says whether every failed build the verdict of vkey names is done again, and forgets it: one retake each.
func (ch *Checker) takeRebuilt(vkey string) bool {
ch.mu.Lock()
defer ch.mu.Unlock()
was := ch.rebuilt[vkey]
delete(ch.rebuilt, vkey)
return was
}
@@ -3,12 +3,9 @@ package check
import (
"context"
"errors"
"maps"
"slices"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -33,7 +30,6 @@ func (f *fakeCompositions) Compose(ctx context.Context, c *Case) (delivery.Compo
// 1297/AC3 (its unit half): a check whose tests pass takes the assigner's composition of it, by the check's own key,
// into its verdict: a pass says it passed with the mesh-sdk and machines it composed, and is no longer "not run".
func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
t.Run("a change to a build module or base names the modules built on it", aChangeToABuildModuleOrBaseNamesTheModulesBuiltOnIt)
b := newBench(t)
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
b.ex.Compositions = f
@@ -103,165 +99,3 @@ func TestACompositionNotHadIsNotRun(t *testing.T) {
type fixedTrunks struct{ heads map[string]string }
func (f fixedTrunks) Heads() map[string]string { return f.heads }
// fakeEnds answers the builder's builds of a candidate as fakeBuilds holds them, recording the builds waited on; the
// composition must not have been asked yet when it is (ADR 0603 §10: the check composes with the builds).
type fakeEnds struct {
builds fakeBuilds
asked map[string]string
composed *fakeCompositions
early bool
}
func (f *fakeEnds) Ended(_ context.Context, builds map[string]string) (map[string]delivery.BuilderBuild, error) {
f.asked = builds
f.early = f.composed != nil && len(f.composed.asked) > 0
out := map[string]delivery.BuilderBuild{}
for module, fp := range builds {
if b, ok := f.builds[module+" "+fp]; ok && (b.State == delivery.BuildDone || b.State == delivery.BuildFailed) {
out[module] = b
}
}
return out, nil
}
func (f *fakeEnds) Done(context.Context, map[string]string) bool { return false }
// fixedServing are the verbs the running builds serve.
type fixedServing []string
func (f fixedServing) Served(context.Context) (func(string) bool, error) {
return func(v string) bool { return slices.Contains(f, v) }, nil
}
// knownRuns are the modules the trunk holds a record of (the indexer's modules): any commit.
type knownRuns []string
func (k knownRuns) CommitOf(module string) string {
if slices.Contains(k, module) {
return trunkA
}
return ""
}
// ADR 0603 §10: before it composes, the check waits on the builder's builds of the candidate's changed modules and composes
// with them; a failed build fails the check naming it, and one not ended within the bound is named. It does not wait on a
// module built by a build module the change alters, nor on one standing on a base the change alters, nor on one the trunk
// holds no record of, and names each in the composition's warnings (and so the check's line); a change that alters none of
// these says nothing.
func aChangeToABuildModuleOrBaseNamesTheModulesBuiltOnIt(t *testing.T) {
dig := func(c string) string { return strings.Repeat(c, 64) }
fp := "src2:" + dig("f")
goBuild := &catalogue.Build{Artifacts: []catalogue.Artifact{{Name: "bin", Kind: catalogue.ArtifactBundle, Language: "go"}}}
tsBuild := &catalogue.Build{Artifacts: []catalogue.Artifact{{Name: "app", Kind: catalogue.ArtifactBundle, Language: "typescript"}}}
done := delivery.BuilderBuild{State: delivery.BuildDone}
for _, row := range []struct {
name string
manifest map[string]catalogue.Manifest
builds map[string]delivery.BuilderBuild // by module, at fp; nil leaves the check without the builds' ends
runs knownRuns // nil: the trunk holds every module
want []string
waited string // the modules waited on, sorted
failed string // the failure's test, when the check fails on a build
tests bool // the tests fail: the check fails before it would wait
served []string // what the running builds serve (ADR 0614 §7); nil reads nothing of what runs
}{
{name: "the Go build module invokes a verb no running build serves",
manifest: map[string]catalogue.Manifest{"mesh-build-go": {Module: "mesh-build-go", Build: goBuild,
Invokes: []string{"seat:package-registry.publish", "seat:artifact-store.put"}}, "web": {Module: "web", Build: goBuild}},
builds: map[string]delivery.BuilderBuild{"mesh-build-go": done}, served: []string{"seat:artifact-store.put"},
want: []string{"the changed build module mesh-build-go invokes seat:package-registry.publish, which no running build serves, " +
"so the check cannot run it: web, which it builds, are composed from their manifests (ADR 0614 §7)"}, waited: "mesh-build-go"},
{name: "the Go build module changes",
manifest: map[string]catalogue.Manifest{"mesh-build-go": {Module: "mesh-build-go", Build: goBuild}, "web": {Module: "web", Build: goBuild},
"mail": {Module: "mail", Build: goBuild}, "docs": {Module: "docs"}},
builds: map[string]delivery.BuilderBuild{"mesh-build-go": done, "docs": done},
want: []string{"the build module mesh-build-go: mail, web, which it builds"}, waited: "docs, mesh-build-go"},
{name: "the Node build module changes",
manifest: map[string]catalogue.Manifest{"mesh-build-node": {Module: "mesh-build-node", Build: goBuild}, "panel": {Module: "panel", Build: tsBuild},
"web": {Module: "web", Build: goBuild}},
builds: map[string]delivery.BuilderBuild{"mesh-build-node": done, "web": done},
want: []string{"the build module mesh-build-node: panel, which it builds"}, waited: "mesh-build-node, web"},
{name: "a base changes",
manifest: map[string]catalogue.Manifest{"image": {Module: "image"},
"web": {Module: "web", Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: "image", Artifact: "img"}}}}},
builds: map[string]delivery.BuilderBuild{"image": done},
want: []string{"alters image, a base of web", "waits on the build of image alone"}, waited: "image"},
{name: "neither changes", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}},
builds: map[string]delivery.BuilderBuild{"web": done}, waited: "web"},
{name: "a build fails", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}, "mail": {Module: "mail", Build: goBuild}},
builds: map[string]delivery.BuilderBuild{"web": {State: delivery.BuildFailed, Error: "go build: undefined: x"}, "mail": done},
waited: "mail, web", failed: "build: web"},
{name: "the tests fail", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}},
builds: map[string]delivery.BuilderBuild{"web": {State: delivery.BuildBuilding}}, tests: true,
want: []string{"composes at once, without waiting on the builds of web"}},
{name: "a build does not end within the bound", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}},
builds: map[string]delivery.BuilderBuild{"web": {State: delivery.BuildBuilding}},
want: []string{"the builds of web did not end within the build-wait-bound"}, waited: "web"},
{name: "a module the trunk does not hold", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}, "new": {Module: "new", Build: goBuild}},
builds: map[string]delivery.BuilderBuild{"web": done}, runs: knownRuns{"web"},
want: []string{"new: the trunk holds no record of it yet"}, waited: "web"},
{name: "the builds cannot be heard", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}},
want: []string{"the builds of web cannot be heard"}},
} {
b := candidateBench(t, true)
if row.tests {
b.test = func(c stepCall) (any, error) {
r := captured[Tested](t, "go-test-failing")
r.Level, _ = c.args["level"].(string)
return r, nil
}
}
comps := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "x", Machines: []string{"g14"}}}
b.ex.Compositions = comps
var ends *fakeEnds
if row.builds != nil {
ends = &fakeEnds{builds: fakeBuilds{}, composed: comps}
for module, build := range row.builds {
ends.builds[module+" "+fp] = build
}
b.ex.BuildEnds = ends
}
if row.runs != nil {
b.ex.Runs = row.runs
}
if row.served != nil {
b.ex.Serving = fixedServing(row.served)
}
read, fps := fixedManifests{}, map[string]delivery.CandidateModule{}
for module, m := range row.manifest {
d := dig(string(rune('a' + len(read))))
read[d], fps[module] = m, delivery.CandidateModule{FP2: fp, Manifest: d}
}
b.ex.Read = read
p := pullOf(catalogueRepository, 9, head1)
p.Fps, p.Modules = fps, slices.Sorted(maps.Keys(fps))
v := b.judge(t, caseOf(t, p)).Verdict
if v.Composition == nil || len(comps.asked) != 1 {
t.Fatalf("%s: composed %d times: %+v", row.name, len(comps.asked), v)
}
if ends != nil {
if got := strings.Join(slices.Sorted(maps.Keys(ends.asked)), ", "); got != row.waited || ends.early {
t.Errorf("%s: waited on %q (after the composition: %v), want %q before it composes", row.name, got, ends.early, row.waited)
}
}
switch {
case row.failed != "" && (v.State != delivery.VerdictFail || v.Failed.Test != row.failed || !strings.Contains(v.Failed.Words, "undefined: x") ||
!strings.Contains(v.Failed.Words, "the check asks itself again when the build is done")):
t.Errorf("%s: the verdict is %s failing %+v, want it failed naming %s, the builder's words and how it is asked again", row.name, v.State, v.Failed, row.failed)
case row.tests && v.State != delivery.VerdictFail:
t.Errorf("%s: %+v, want the tests' failure", row.name, v)
case row.failed == "" && !row.tests && v.State != delivery.VerdictPass:
t.Errorf("%s: %+v", row.name, v)
}
words := compositionWords(v.Composition, v.Took.Composition)
for _, w := range row.want {
if !strings.Contains(words, w) {
t.Errorf("%s: the line says %q, want it to name %q", row.name, words, w)
}
}
if row.want == nil && v.Composition.Words != "" {
t.Errorf("%s: warned %q though the check waited on every build of the change", row.name, v.Composition.Words)
}
}
}
@@ -52,12 +52,7 @@ type EngineCompositions struct {
// Manifests are the indexer's trunk manifest digests by module, and Verdicts its refusals of them: the catalogue the
// sandbox composes with, before the case's own manifests are laid over.
Manifests TrunkManifests
// Published names the latest published version of every mesh package (Examiner.Published): the composer is built
// against them and the group's heads, as a check is (issue 1921); nil names the group's heads alone.
Published interface {
LatestPackages() delivery.PackageVersions
}
Verdicts func() map[string]delivery.ManifestVerdict
Verdicts func() map[string]delivery.ManifestVerdict
}
// composerHead is the node-engine member's head of a case, or empty.
@@ -152,12 +147,6 @@ func JudgeEngines(group, heads, composerName string, nodes []string, answers map
v.Warnings = append(v.Warnings, node+": "+delivery.ComposeNoAnswer)
continue
}
if a.Verdict == delivery.ComposeCandidateNotJudged {
// Composed in the sandbox and not judged (ADR 0593 §9): named as no answer, with the composer's words; decided by the
// verdict, never by the words (ADR 0601 §4's kind).
v.Warnings = append(v.Warnings, node+": "+delivery.ComposeNoAnswer+" ("+a.Refused+")")
continue
}
v.Machines = append(v.Machines, node)
if a.Verdict != delivery.ComposeCandidateRefused {
continue
@@ -225,18 +214,15 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s
if err != nil {
return a, err
}
job := delivery.ComposeJob{Fetch: withPackages(map[string]any{"repository": nodeEngineRepo, "commit": head}, packagesFor(e.Published, c, nodeEngineRepo)), Dir: delivery.RootPath,
job := delivery.ComposeJob{Fetch: map[string]any{"repository": nodeEngineRepo, "commit": head}, Dir: delivery.RootPath,
Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
verdict, err := e.Sandbox.Compose(ctx, job)
if err != nil {
return a, fmt.Errorf("the composer at the node-engine's head %s did not answer for %s: %w", head, node, err)
}
out := delivery.ComposeCandidateAnswer{V: a.V, Node: node, Candidate: a.Candidate, Verdict: delivery.ComposeCandidateComposed}
notJudged := ""
for _, cv := range verdict.Cases {
if m, ok := cv.Machines[node]; ok && m.NotJudged != "" {
notJudged = m.NotJudged // the composer did not judge the node: neither composed nor refused (ADR 0593 §9)
} else if ok && !m.Composes {
if m, ok := cv.Machines[node]; ok && !m.Composes {
out.Verdict = delivery.ComposeCandidateRefused
out.Refused = strings.Join(append(append([]string{m.Reason}, m.Problems...), m.Refusals...), "; ")
}
@@ -248,8 +234,5 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s
}
}
sort.Slice(out.Modules, func(i, j int) bool { return out.Modules[i].Module < out.Modules[j].Module })
if notJudged != "" && out.Verdict == delivery.ComposeCandidateComposed {
out.Verdict, out.Refused = delivery.ComposeCandidateNotJudged, notJudged
}
return out, nil
}
@@ -9,7 +9,6 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -151,7 +150,7 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
Machines: map[string]composer.Machine{node: {Composes: false, Reason: "the new composer refuses it"}}}}}
}}
ec := EngineCompositions{Asker: e, Placed: func() ([]string, error) { return []string{"anchor"}, nil }, Bound: time.Second,
Sandbox: sb, Manifests: digests{"web": "sha256:" + strings.Repeat("c", 64)}, Published: published{}}
Sandbox: sb, Manifests: digests{"web": "sha256:" + strings.Repeat("c", 64)}}
v, err := ec.Compose(context.Background(), host)
if err != nil {
t.Fatal(err)
@@ -166,32 +165,9 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
string(req.Input) != string(withInput.Input) || req.Cases[0].Manifests["web"].Digest == "" {
t.Errorf("job = %+v, request %+v", job, req)
}
// Issue 1921: the composer is built at the group's heads, also a member that publishes no package yet.
if pkgs, _ := buildstep.PackagesArg(job.Fetch["packages"]); len(pkgs) != 1 || pkgs[delivery.ForgeModulePrefix+"mesh-delivery"] != head1 {
t.Errorf("the composer is fetched with the packages %v, want the group's mesh-delivery member at its head %s alone", pkgs, head1)
}
if v.State != delivery.CompositionFail || v.SDK != head2 || !strings.Contains(v.Words, "the new composer refuses it") {
t.Errorf("verdict = %+v", v)
}
// ADR 0593 §9 (ADR 0601 §4's kind): a node the composer did not judge is named as no answer with its words, by the
// contract's field, and the check passes; words in Reason alone are no such mark.
sb.err = nil
for _, row := range []struct {
m composer.Machine
pass bool
}{
{composer.Machine{NotJudged: "its node-engine sends no running state for web"}, true},
{composer.Machine{Reason: "not judged: its node-engine sends no running state for web"}, false},
} {
sb.v = func(node string) composer.Verdict {
return composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: "change", Machines: map[string]composer.Machine{node: row.m}}}}
}
v, err = ec.Compose(context.Background(), host)
named := len(v.Warnings) == 1 && strings.Contains(v.Warnings[0], "anchor: "+delivery.ComposeNoAnswer+" (its node-engine sends no running state for web)")
if err != nil || (v.State == delivery.CompositionPass) != row.pass || named != row.pass {
t.Errorf("%+v: verdict %+v, %v", row.m, v, err)
}
}
// Review finding 3 (ADR 0593 §9 lets only a node that does not answer pass): a composer at the head that cannot
// run fails the composition, naming the head, never passing as a silent node.
sb.err = errors.New("the build of mesh-compose failed")
@@ -20,8 +20,6 @@ import (
gobuild "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -1002,17 +1000,6 @@ func (k keptList) PredefinedList(context.Context) (json.RawMessage, error) {
return json.RawMessage(k), nil
}
// fixedManifests are the indexer's kept manifests by digest as a test gives them.
type fixedManifests map[string]catalogue.Manifest
func (f fixedManifests) Manifest(_ context.Context, digest string) (catalogue.Manifest, error) {
m, ok := f[digest]
if !ok {
return catalogue.Manifest{}, errors.New("no manifest " + digest)
}
return m, nil
}
// fixedDigests are the indexer's manifest digests as a test gives them.
type fixedDigests map[string]string
@@ -1030,75 +1017,17 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
// untouched is the judge's refusal of mail, which the change leaves as it is, and trunkMail its refusal on the trunk:
// a problem the trunk had is a warning, one the change causes fails (ADR 0579 §2)
untouched, trunkMail string
// cycle reads the manifests, so the change's module graph is worked out (ADR 0603 §2): with "provisions" the change's
// web awaits mail's site while mail awaits web's database, and trunkCycle has the trunk's web await it too; with "bases"
// each is built on the other; with "verbs" each invokes the other's tool, and with "a verb and a provision" web invokes
// mail's tool while mail awaits web's database: both left out, so neither is refused (ADR 0614 §3, §6).
cycle string
trunkCycle bool
// indexer makes web the holder of the seat mesh-catalog, and the trunk's manifest validator answers for it: "held" with no verdict,
// "aside" with a field it does not know; ownJudge has the group change the judge, so the manifest is also judged by the
// trunk's, in a second ask (ADR 0633 clause 6, 1585/T22, T23)
indexer string
ownJudge bool
// try judges the same change as a whole group try, whose manifests come from the indexer's candidate of the try's
// group (issue 1921): the same judge, the same verdicts.
try bool
}{
{name: "accepted"},
{name: "accepted in a group try", try: true},
{name: "refused in a group try", try: true, refused: "web has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web has no id`},
{name: "a cycle the group try makes", try: true, cycle: "provisions", fails: "the change's module graph has the cycle mail -> web -> mail"},
{name: "the indexer accepted by the trunk's manifest validator", indexer: "ok"},
{name: "the indexer held by the trunk's manifest validator", indexer: "held", fails: "the indexer web, the holder of the seat mesh-catalog, would never run: the trunk's manifest validator at mesh-delivery's trunk " + strings.Repeat("d", 40) + " does not accept its manifest: the manifest validator holds it, with no verdict yet"},
{name: "the indexer with a field the trunk's manifest validator does not know", indexer: "aside", fails: "the manifest validator does not know its fields new_field, so it waits"},
{name: "the indexer refused by the trunk's manifest validator", indexer: "refused", refused: "web has no id", fails: "does not accept its manifest: web has no id"},
{name: "the indexer under a changed judge is judged by the trunk's as well", indexer: "aside", ownJudge: true, fails: "the manifest validator does not know its fields new_field, so it waits"},
{name: "a touched manifest the judge holds", indexer: "heldplain", fails: "holds the manifest of web: it gave no verdict"},
{name: "an untouched manifest's problem the trunk had", untouched: "mail says no credential form", trunkMail: "mail says no credential form"},
{name: "a problem the change causes in an untouched manifest", untouched: "mail reads a state web no longer keeps",
fails: "refuses the manifest of mail: mail reads a state web no longer keeps"},
{name: "a cycle the change makes", cycle: "provisions", fails: "the change's module graph has the cycle mail -> web -> mail"},
{name: "a cycle the trunk has", cycle: "provisions", trunkCycle: true},
{name: "a cycle of bases", cycle: "bases", fails: "the change's module graph has the cycle mail -> web -> mail"},
{name: "a cycle of verbs", cycle: "verbs"},
{name: "a cycle of a verb and a provision", cycle: "a verb and a provision"},
{name: "refused", refused: "web requires \"amqp\"\nweb has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web requires "amqp"; web has no id`},
} {
b := newBench(t)
b.ex.Manifests = fixedDigests{"mail": strings.Repeat("a", 64), "web": strings.Repeat("b", 64)}
b.ex.Trunks = fixedTrunks{heads: map[string]string{DeliveryRepository: trunk}}
b.ex.Predefined = keptList(`{"font-size": {"type": "number", "description": "the size"}}`)
if row.cycle != "" {
offer := func(name string) []catalogue.Offer {
return []catalogue.Offer{{Name: name, Scope: catalogue.ScopeMesh, AnsweredBy: catalogue.AnsweredByHelper}}
}
on := func(module string) *catalogue.Build {
return &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: module, Artifact: "img"}}}
}
trunkWeb := catalogue.Manifest{Module: "web", Provides: offer("database")}
if row.trunkCycle {
trunkWeb.Requires = []string{"site"}
}
mail := catalogue.Manifest{Module: "mail", Provides: offer("site"), Requires: []string{"database"}}
changed := catalogue.Manifest{Module: "web", Provides: offer("database"), Requires: []string{"site"}}
switch row.cycle {
case "bases":
mail = catalogue.Manifest{Module: "mail", Build: on("web")}
changed = catalogue.Manifest{Module: "web", Build: on("mail")}
case "verbs":
mail = catalogue.Manifest{Module: "mail", Tools: []string{"mail_send"}, Invokes: []string{"web.web_page"}}
changed = catalogue.Manifest{Module: "web", Tools: []string{"web_page"}, Invokes: []string{"mail.mail_send"}}
case "a verb and a provision":
mail = catalogue.Manifest{Module: "mail", Tools: []string{"mail_send"}, Requires: []string{"database"}}
changed = catalogue.Manifest{Module: "web", Provides: offer("database"), Invokes: []string{"mail.mail_send"}}
}
b.ex.Read = fixedManifests{strings.Repeat("a", 64): mail, strings.Repeat("b", 64): trunkWeb, web: changed}
}
if row.indexer != "" && row.indexer != "heldplain" {
b.ex.Read = fixedManifests{strings.Repeat("a", 64): {Module: "mail"}, strings.Repeat("b", 64): {Module: "web"},
web: {Module: "web", Claims: []catalogue.Claim{{Name: CatalogSeat}}}}
}
var judged []delivery.ComposeJob
answer := b.asks.Answer
b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
@@ -1107,12 +1036,6 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
}
judged = append(judged, *a.Compose)
read := composer.ManifestRead{Accepted: row.refused == "", Refused: row.refused}
switch row.indexer {
case "held", "heldplain":
read = composer.ManifestRead{}
case "aside":
read.SetAside = "new_field"
}
verdict, _ := json.Marshal(composer.Verdict{V: composer.Version, SDK: "exe:1", Cases: []composer.CaseVerdict{{Name: JudgeCase,
Manifests: map[string]composer.ManifestRead{"web": read, "mail": {Accepted: row.untouched == "", Refused: row.untouched}}},
{Name: JudgeTrunk, Manifests: map[string]composer.ManifestRead{"web": {Accepted: true}, "mail": {Accepted: row.trunkMail == "", Refused: row.trunkMail}}}}})
@@ -1120,53 +1043,9 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
return delivery.Step{ID: a.ID, State: delivery.StepPassed, Result: result}
}
p := pullOf(repoA, 7, head1)
if row.ownJudge {
p = pullOf(DeliveryRepository, 7, head1)
b.fetch.Changed = []string{JudgeDir + "/judge/judge.go"}
}
p.Modules = []string{"web"}
p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}}
// Issue 1921: the judge is built at the group's heads too, also a member that publishes no package yet.
cs := caseOf(t, p, pullOf("novox/mesh-networking", 9, head2))
if row.try {
cs = TryCase("try-1", Try{Members: []TryMember{{Repository: repoA, Head: head1, Trunk: trunkA}}}, nil)
b.ex.Candidates = &fakeTryCandidates{cand: delivery.Candidate{FPSet: "fps", Pulls: []delivery.CandidatePull{p}}}
}
v := b.judge(t, cs).Verdict
if row.indexer == "ok" && !slices.ContainsFunc(cs.notes, func(n string) bool {
return strings.Contains(n, "the new indexer's manifest of web is accepted by the trunk's manifest validator")
}) {
t.Errorf("%s: the case says %v, want the new indexer's manifest said accepted by the trunk's manifest validator", row.name, cs.notes)
}
// Issue 1921: the judge of the change is built at the group's heads (also a member that publishes no package yet), but
// the trunk's manifest validator that rules on a new indexer is built at the latest published versions alone, never
// at a group's head (ADR 0633 clause 6): a group with a member of its own then judges the indexer a second time.
trunkRuns := 0
if row.indexer != "" && row.indexer != "heldplain" && (row.ownJudge || !row.try) {
trunkRuns = 1
}
if want := 1 + trunkRuns; len(judged) != want {
t.Fatalf("%s: %d judge asks %+v, want %d", row.name, len(judged), judged, want)
}
for i, j := range judged {
pkgs, _ := buildstep.PackagesArg(j.Fetch["packages"])
atGroup := pkgs[delivery.ForgeModulePrefix+"mesh-networking"] == head2
switch trunkRun := trunkRuns == 1 && i == len(judged)-1; {
case row.try:
case trunkRun && atGroup:
t.Errorf("%s: the trunk's manifest validator is built with the group's head: %v", row.name, pkgs)
case !trunkRun && !atGroup:
t.Errorf("%s: the judge at %v is fetched with the packages %v, want the group's mesh-networking member at its head %s", row.name, j.Fetch["commit"], pkgs, head2)
}
}
if row.ownJudge && (len(judged) != 2 || judged[0].Fetch["commit"] != head1 || judged[1].Fetch["commit"] != trunk) {
t.Fatalf("%s: judged %+v, want the head's judge and then the trunk's", row.name, judged)
}
if row.ownJudge {
judged = judged[1:]
} else if trunkRuns == 1 {
judged = judged[1:] // the group's run; the trunk's is the one read below
}
v := b.judge(t, caseOf(t, p)).Verdict
if len(judged) != 1 || judged[0].Fetch["repository"] != DeliveryRepository || judged[0].Fetch["commit"] != trunk ||
judged[0].Package != JudgePackage || judged[0].Dir != JudgeDir {
t.Fatalf("%s: judged %+v (verdict %+v)", row.name, judged, v)
@@ -1,76 +0,0 @@
package check
import (
"context"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/composer"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
// The check refuses a change whose module graph has a cycle and names it (novox/hq ADR 0603 §2): modules that would each
// wait for the other for good, through their bases, the provisions and bus state they await, or the verb edges the graph
// keeps (ADR 0614 §6). A verb edge on a cycle is left out of the graph, so it is never refused (§3), and a package or a
// build module's edge is a trigger, never a wait, so a cycle through one is not refused either (§6). The graph is worked out from the
// catalogue as the change leaves it (every trunk manifest, each one a pull request touches replaced by its candidate's),
// with the contract's ManifestGraph, the way the indexer works out its graph's edges. A cycle the trunk already has among
// the same modules is the trunk's: said in the verdict's notes, not the change's failure.
// ManifestReader reads a kept manifest by its digest (the indexer's manifests state).
type ManifestReader interface {
Manifest(ctx context.Context, digest string) (catalogue.Manifest, error)
}
// refuseCycle fails the check for a cycle of the change's module graph, naming it; nil when the case touches no manifest,
// when nothing reads manifests, or when there is none. A manifest that cannot be read fails the check, said, never passes it.
func (st *state) refuseCycle(ctx context.Context) *delivery.Verdict {
if st.j.Manifests == nil || st.j.Read == nil || st.narrowTry() {
return nil
}
trunk := st.j.Manifests.ManifestDigests()
req, touched := judgeRequest(trunk, st.c.Pulls)
if len(touched) == 0 {
return nil
}
read := func(digests map[string]composer.Manifest) (map[string]catalogue.Manifest, error) {
out := map[string]catalogue.Manifest{}
for _, module := range slices.Sorted(maps.Keys(digests)) {
m, err := st.j.Read.Manifest(ctx, digests[module].Digest)
if err != nil {
return nil, fmt.Errorf("the manifest %s of %s: %w", digests[module].Digest, module, err)
}
out[module] = m
}
return out, nil
}
change, err := read(req.Cases[0].Manifests)
if err != nil {
return fail(goModule, "the change's module graph cannot be worked out, so no cycle can be ruled out: "+err.Error())
}
cycle := graph.OfManifests(change).Cycle()
if cycle == nil {
st.c.note("the change's module graph has no cycle")
return nil
}
words := strings.Join(cycle, " -> ")
if before, err := read(req.Cases[1].Manifests); err == nil {
among := graph.Graph{}
for name, n := range graph.OfManifests(before) {
if slices.Contains(cycle, name) {
among[name] = n
}
}
if among.Cycle() != nil {
st.c.note("the module graph has the cycle " + words + ", which the trunk already has among the same modules (ADR 0603 §2)")
return nil
}
}
return fail(goModule, fmt.Sprintf("the change's module graph has the cycle %s: these modules would each wait for the other for "+
"good, through their bases, the provisions and bus state they await, or the verbs they invoke (ADR 0603 §2, ADR 0614 §6)", words))
}
@@ -102,11 +102,6 @@ type Examiner struct {
// Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing
// of the builds.
Builds BuildStates
// BuildEnds waits on the builder's builds of the candidate before the check composes (ADR 0603 §10); nil composes
// without waiting, said in the composition's warnings.
BuildEnds BuildEnds
// BuildWait is how long the check waits for those builds (the setting build-wait-bound); DefaultBuildWait when 0.
BuildWait time.Duration
// Announce says a preview was written under the key (EventPreviewWritten); nil says nothing.
Announce Announcer
// Packages names, of a repository, the paths (repository-relative) its TypeScript package artifacts are built from, from
@@ -119,15 +114,6 @@ type Examiner struct {
// Manifests are the indexer's manifest digests by module: the catalogue a change's manifests are judged in by the
// indexer's judge (manifests_judge.go, novox/hq ADR 0578 §2); nil judges no manifest.
Manifests TrunkManifests
// Read reads a kept manifest by its digest: the change's module graph is worked out from them (graph_cycle.go, novox/hq
// ADR 0603 §2); nil works out no graph.
Read ManifestReader
// Serving says, from what runs, whether a running healthy build serves a seat verb or module tool (novox/hq ADR 0614
// §7): the node-engines' haves, the builder's build of each, its manifest (graph.Served). Nil reads nothing, and no
// changed build module is named for a verb it invokes.
Serving interface {
Served(ctx context.Context) (func(invoke string) bool, error)
}
// Predefined reads the catalogue's predefined keys the indexer keeps (manifests_judge.go); nil reads none.
Predefined PredefinedList
// Compositions makes a check's composition (ADR 0593 §7): the engines' answers (EngineCompositions); nil leaves the
@@ -139,18 +125,8 @@ type Examiner struct {
Nodes interface {
Names(ctx context.Context) ([]string, error)
}
// Candidates answers the indexer's candidate of a try's group once it wrote it from the try's previews, the manifests
// and fingerprints a group try's manifest validator and composition judge (issue 1921); nil leaves a whole try with no
// manifest or composition judged, said.
Candidates TryCandidates
Now func() time.Time
Logf func(format string, args ...any)
}
// TryCandidates answers the candidate the indexer holds for a try's group at its heads hash, waiting for it within the
// bound; an error says it did not come (and why).
type TryCandidates interface {
TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error)
Now func() time.Time
Logf func(format string, args ...any)
}
// CompositionReader makes a check's composition verdict; an error is only a case that cannot be composed at all.
@@ -256,50 +232,6 @@ type state struct {
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
unpreviewed []string
// noCandidate says why a whole try has no candidate of its group from the indexer, so its manifests were not judged and
// it was not composed; empty when it has one, and for a pull request's check.
noCandidate string
}
// narrowTry says the case is a try narrowed to a level or to some tests: it judges no manifest, graph or composition.
func (st *state) narrowTry() bool { return st.c.Try != nil && !st.c.Try.Whole() }
// takeTryCandidate gives a whole try's pulls the fingerprints and manifests the indexer's candidate of its group holds
// (the group's pull-request check has them from its candidate, CaseOf), once the indexer wrote it from the try's previews.
// A try with no such candidate is said so, in a note and in its composition, and is no pass (Tryer.run).
func (st *state) takeTryCandidate(ctx context.Context) {
if st.c.Try == nil || !st.c.Try.Whole() {
return
}
no := func(why string) {
st.noCandidate = "the group's manifests and composition were not judged: " + why
st.c.note(st.noCandidate)
}
if st.j.Candidates == nil {
no("the checker reads no candidate of the indexer's")
return
}
if len(st.unpreviewed) > 0 {
no("no preview of a merge of " + strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer)")
return
}
if st.c.Wait != nil {
st.c.Wait("the indexer's candidate of the group at its heads (up to " + st.j.buildWait().String() + ")")
defer st.c.Wait("")
}
cand, err := st.j.Candidates.TryCandidate(ctx, st.c.Group, st.c.HeadsHash, st.j.buildWait())
if err != nil {
no(err.Error())
return
}
for i, p := range st.c.Pulls {
for _, cp := range cand.Pulls {
if cp.Repository == p.Repository && cp.Head.Head == p.Head {
st.c.Pulls[i].Modules, st.c.Pulls[i].Fps = cp.Modules, cp.Fps
}
}
}
st.c.FPSet = cand.FPSet
}
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
@@ -310,8 +242,7 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
tests := st.judgeAll(ctx)
st.takeTryCandidate(ctx)
if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil {
if v := settled(joinFailures(tests, st.judgeManifests(ctx))); v != nil {
return st.finish(ctx, *v), nil
}
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
@@ -337,14 +268,7 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
return
}
if st.narrowTry() {
return
}
if st.noCandidate != "" {
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: st.noCandidate}
return
}
if st.j.Compositions == nil {
if st.j.Compositions == nil || st.c.Try != nil {
return
}
if len(st.unpreviewed) > 0 {
@@ -353,7 +277,6 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer), so the change cannot be composed"}
return
}
words := st.awaitBuilds(ctx, v) // ADR 0603 §10: the check composes with the candidate's builds
comp, err := st.j.Compositions.Compose(ctx, st.c)
if err != nil {
// Said in the verdict, never left as a composition not yet run (review finding 4).
@@ -361,7 +284,6 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: "not made: " + err.Error()}
return
}
comp.Warnings = append(comp.Warnings, words...) // ADR 0603 §10
withComposition(v, comp)
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
@@ -495,7 +417,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
// Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests
// failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0).
switch {
case st.narrowTry(): // a try of a level or some tests owes the indexer no preview: it judges no manifest
case st.c.Try != nil: // a try owes the indexer no preview: no pull request holds its head
case len(merged) == len(modules):
st.j.writePreview(ctx, p, merged)
default:
@@ -624,7 +546,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch))
}
if len(t.fetched.Conflicts) > 0 {
if !st.narrowTry() {
if st.c.Try == nil {
st.j.writePreview(ctx, p, append(*merged, t.fetched))
}
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
@@ -97,7 +97,7 @@ func judgeRequest(trunk map[string]string, pulls []Pull) (req composer.Request,
// refuses. A case that touches no manifest, or an examiner that reads no trunk manifests, judges nothing; a judge that
// cannot be asked or does not answer fails the check, said, never passes it.
func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
if st.j.Manifests == nil || st.narrowTry() {
if st.j.Manifests == nil || st.c.Try != nil {
return nil
}
req, touched := judgeRequest(st.j.Manifests.ManifestDigests(), st.c.Pulls)
@@ -120,12 +120,26 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
req.Predefined = list
st.c.note("the manifests are judged against the predefined keys of " + from)
}
cv, trunkCase, v := st.runJudge(ctx, req, touched, commit, at, st.packagesOf(DeliveryRepository))
if v != nil {
return v
raw, err := json.Marshal(req)
if err != nil {
return fail(goModule, err.Error())
}
if v := st.refuseUnmovableIndexer(ctx, req, touched, cv, at); v != nil {
return v
job := delivery.ComposeJob{Fetch: map[string]any{"repository": DeliveryRepository, "commit": commit}, Dir: JudgeDir,
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Compose: &job}
st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "")
st.c.Asked(a.ID)
if err := st.j.Asks.Put(ctx, a); err != nil {
return fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error())
}
step, err := st.j.Asks.Wait(ctx, a)
if err != nil {
return fail(goModule, "the judge of the change's manifests: "+err.Error())
}
cv, trunkCase, why := judgeAnswer(step)
if why != "" {
return fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why))
}
// A new or changed manifest is judged in full; of one the change leaves as it is, a problem the trunk already had is a
// warning (it is not refused for a rule it already broke), and a problem the change causes in it, which the trunk case
@@ -133,13 +147,7 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
var refused, warned []string
for _, module := range slices.Sorted(maps.Keys(cv.Manifests)) {
r := cv.Manifests[module]
if r.Accepted {
continue
}
if r.Refused == "" { // held: a verdict not given is not one given (ADR 0633 clause 6)
if slices.Contains(touched, module) {
refused = append(refused, fmt.Sprintf("the indexer's judge at %s holds the manifest of %s: it gave no verdict", at, module))
}
if r.Accepted || r.Refused == "" {
continue
}
had := map[string]bool{}
@@ -175,100 +183,6 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
return nil
}
// runJudge asks the Go build seat to build the judge at the commit and run it over the request, and reads its change and
// trunk cases; a failed verdict when it cannot be asked or does not answer.
func (st *state) runJudge(ctx context.Context, req composer.Request, touched []string, commit, at string, versions delivery.PackageVersions) (change, trunk composer.CaseVerdict, v *delivery.Verdict) {
raw, err := json.Marshal(req)
if err != nil {
return change, trunk, fail(goModule, err.Error())
}
job := delivery.ComposeJob{Fetch: withPackages(map[string]any{"repository": DeliveryRepository, "commit": commit}, versions), Dir: JudgeDir,
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
At: st.j.now().UTC(), Compose: &job}
st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "")
st.c.Asked(a.ID)
if err := st.j.Asks.Put(ctx, a); err != nil {
return change, trunk, fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error())
}
step, err := st.j.Asks.Wait(ctx, a)
if err != nil {
return change, trunk, fail(goModule, "the judge of the change's manifests: "+err.Error())
}
change, trunk, why := judgeAnswer(step)
if why != "" {
return change, trunk, fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why))
}
return change, trunk, nil
}
// CatalogSeat is the seat the indexer holds: the module whose manifest claims it is the indexer, found by the claim and
// never by its name (novox/hq ADR 0633 clause 6, ADR 0601 §4).
const CatalogSeat = "mesh-catalog"
// refuseUnmovableIndexer fails the check when the trunk's manifest validator, the one the running parts hold, would
// refuse or hold the change's new indexer manifest: such an indexer would never run, and every revision behind it would
// wait for ever (ADR 0633 clause 6). The indexer is the touched module whose manifest claims the seat mesh-catalog.
// When the change judges with its own head's manifest validator (it changes it), the manifest is judged by the trunk's
// as well. The verdicts read are the trunk's manifest validator's: its change case, where refused, held (not accepted
// with no reason) or with a field set aside (a field the running manifest validator does not know waits, ADR 0601 §3)
// each fail it, naming the module and the problems.
func (st *state) refuseUnmovableIndexer(ctx context.Context, req composer.Request, touched []string, cv composer.CaseVerdict, judgedAt string) *delivery.Verdict {
if st.j.Read == nil {
return nil
}
var indexer string
for _, module := range touched {
m, err := st.j.Read.Manifest(ctx, req.Cases[0].Manifests[module].Digest)
if err != nil {
return fail(goModule, fmt.Sprintf("the manifest of %s cannot be read, so the indexer cannot be ruled out: %v", module, err))
}
for _, c := range m.Claims {
if c.Name == CatalogSeat {
indexer = module
}
}
}
if indexer == "" {
return nil
}
// The validator the running parts hold is the trunk's, built at the latest published versions alone, never at the group's
// heads (ADR 0633 clause 6): a group's mesh-sdk or mesh-contracts head must not change the validator that judges the new
// indexer. The head's run is reused when it was that validator already.
if trunkVersions := st.trunkJudgeVersions(); st.judgeHead != "" || !maps.Equal(st.packagesOf(DeliveryRepository), trunkVersions) {
trunk := ""
if st.j.Trunks != nil {
trunk = st.j.Trunks.Heads()[DeliveryRepository]
}
if trunk == "" {
return fail(goModule, "the new indexer cannot be judged by the trunk's manifest validator: the trunk head of "+DeliveryRepository+" is not known")
}
var v *delivery.Verdict
judgedAt = "mesh-delivery's trunk " + trunk
if cv, _, v = st.runJudge(ctx, req, touched, trunk, judgedAt, trunkVersions); v != nil {
return v
}
}
r, ok := cv.Manifests[indexer]
var problems []string
switch {
case !ok:
problems = append(problems, "the manifest validator gave no verdict")
case r.Refused != "":
problems = append(problems, strings.Split(r.Refused, "\n")...)
case !r.Accepted:
problems = append(problems, "the manifest validator holds it, with no verdict yet")
case r.SetAside != "":
problems = append(problems, "the manifest validator does not know its fields "+r.SetAside+", so it waits")
}
if len(problems) > 0 {
return fail(goModule, fmt.Sprintf("the indexer %s, the holder of the seat %s, would never run: the trunk's manifest validator at %s does not accept its manifest: %s",
indexer, CatalogSeat, judgedAt, strings.Join(problems, "; ")))
}
st.c.note(fmt.Sprintf("the new indexer's manifest of %s is accepted by the trunk's manifest validator at %s", indexer, judgedAt))
return nil
}
// judgeAnswer reads the judge's change and trunk cases from its step, or why it cannot.
func judgeAnswer(step delivery.Step) (change, trunk composer.CaseVerdict, why string) {
if step.State != delivery.StepPassed {
@@ -343,9 +257,3 @@ func (st *state) judgedByMember(p Pull, merged []Fetched) {
}
}
}
// trunkJudgeVersions are the versions the trunk's manifest validator is built at: the latest published version of each
// package, with no group member's head (ADR 0633 clause 6).
func (st *state) trunkJudgeVersions() delivery.PackageVersions {
return packagesFor(st.j.Published, &Case{}, DeliveryRepository)
}
@@ -3,8 +3,8 @@ package check
import (
"context"
"errors"
"maps"
"git.novox.be/novox/mesh-delivery/sdk/grouppackages"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -31,28 +31,37 @@ type Forge interface {
File(ctx context.Context, repository, path, ref string) (string, error)
}
// packagesOf is the version of each mesh package a Go build of repository (a check ask, the judge, the composer) builds
// against (ADR 0573 §2, §4): grouppackages.Heads over the latest published version of each package the mesh's modules use
// (Published) and the heads of the case's pull requests, so a member that publishes no package yet is named too (issue
// 1921). Nil for none.
// packagesOf is the version of each mesh package a Go check of repository builds against (ADR 0573 §2, §4): the latest
// published version of each package the mesh's modules use (Published), and for a package one of the case's pull
// requests publishes, that pull request's head (mesh-sdk's Go module whenever mesh-sdk is a member). A package of
// repository itself comes with the tree the check fetched, and is left out. Nil for none.
func (st *state) packagesOf(repository string) delivery.PackageVersions {
return packagesFor(st.j.Published, st.c, repository)
}
// packagesFor is packagesOf for a case and the published versions: every producer of a build at a group's heads takes its
// versions from here, so none leaves a member out.
func packagesFor(published interface {
LatestPackages() delivery.PackageVersions
}, c *Case, repository string) delivery.PackageVersions {
var latest delivery.PackageVersions
if published != nil {
latest = published.LatestPackages()
versions := delivery.PackageVersions{}
if st.j.Published != nil {
maps.Copy(versions, st.j.Published.LatestPackages())
}
heads := map[string]string{}
for _, m := range c.Pulls {
heads[m.Repository] = m.Head
if st.c.member(SDKRepository) != nil {
if _, named := versions[SDKModulePath]; !named {
versions[SDKModulePath] = ""
}
}
return grouppackages.Heads(latest, heads, repository)
for pkg := range versions {
repo, _, ok := delivery.PackageRepository(pkg)
if !ok || delivery.SameRepository(repo, repository) {
delete(versions, pkg)
continue
}
if m := st.c.member(repo); m != nil && len(m.Head) == 40 {
versions[pkg] = m.Head
}
if versions[pkg] == "" {
delete(versions, pkg)
}
}
if len(versions) == 0 {
return nil
}
return versions
}
// unheardWords ends the words of a failed read of the forge that reached nobody: the forge was not heard (UnheardWords).
@@ -71,17 +80,3 @@ func forgeMachine(err error, node string) string {
}
return node
}
// withPackages puts the versions into a fetch's arguments (the fetch tool's packages argument, which a compose ask has no
// field for: a compose ask names none, StepAsk.Packages), so the composer or the judge is built at the group's heads.
func withPackages(fetch map[string]any, versions delivery.PackageVersions) map[string]any {
if len(versions) == 0 {
return fetch
}
packages := make(map[string]any, len(versions))
for pkg, version := range versions {
packages[pkg] = version
}
fetch["packages"] = packages
return fetch
}
@@ -6,7 +6,6 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/sdk/grouppackages"
"git.novox.be/novox/mesh-sdk/go/buildstep"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -61,48 +60,6 @@ func TestAGroupMemberIsBuiltAgainstTheSDKMembersHeadWhateverItsGoModNames(t *tes
if got := packagesOfFetch(t, b, "novox/mesh-catalog"); len(got) != 2 || got[SDKModulePath] != trunkS || got[contract] != trunkA {
t.Errorf("a pull request alone is built against %v, want every package at its latest published version", got)
}
// A member that publishes no package yet is still named, by its root module path, at its head (issue 1921).
b = newBench(t)
b.ex.Published = published{}
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf("novox/mesh-host", 8, head2))).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("an unpublished member: verdict = %+v", v)
}
if got := packagesOfFetch(t, b, repoA); len(got) != 1 || got["git.novox.be/novox/mesh-host"] != head2 {
t.Errorf("the member's packages = %v, want the unpublished member's root at its head %s alone", got, head2)
}
// The function every producer takes the versions from (issue 1921): rows for what a build at a group's heads is asked.
h := func(c string) string { return strings.Repeat(c, 40) }
const (
contracts = "git.novox.be/novox/mesh-contracts/go/clients/assignerclient"
tools = "git.novox.be/novox/mesh-tools/go/x"
)
for _, tc := range []struct {
name string
latest delivery.PackageVersions
heads map[string]string
own string
want delivery.PackageVersions
}{
{"none", nil, nil, "novox/mesh-host", nil},
{"trunk latest alone", delivery.PackageVersions{tools: h("a")}, nil, "novox/mesh-host", delivery.PackageVersions{tools: h("a")}},
{"a member at its head replaces the latest", delivery.PackageVersions{SDKModulePath: h("a")}, map[string]string{SDKRepository: h("b")}, "novox/mesh-host",
delivery.PackageVersions{SDKModulePath: h("b")}},
{"a member that publishes no package yet is named by its root module", delivery.PackageVersions{SDKModulePath: h("a")},
map[string]string{"novox/mesh-contracts": h("c")}, "novox/mesh-host",
delivery.PackageVersions{SDKModulePath: h("a"), "git.novox.be/novox/mesh-contracts": h("c")}},
{"mesh-sdk is named by its go module before any package of it is known", nil, map[string]string{SDKRepository: h("b")}, "novox/mesh-host",
delivery.PackageVersions{SDKModulePath: h("b")}},
{"a member whose packages are known gets no root", delivery.PackageVersions{contracts: h("a")},
map[string]string{"novox/mesh-contracts": h("c")}, "novox/mesh-host", delivery.PackageVersions{contracts: h("c")}},
{"the build's own repository is left out", delivery.PackageVersions{contracts: h("a")},
map[string]string{"novox/mesh-contracts": h("c")}, "novox/mesh-contracts", nil},
{"a head that is no full commit names nothing", nil, map[string]string{"novox/mesh-contracts": "abc"}, "novox/mesh-host", nil},
} {
got := grouppackages.Heads(tc.latest, tc.heads, tc.own)
if !maps.Equal(got, tc.want) {
t.Errorf("%s: got %v, want %v", tc.name, got, tc.want)
}
}
}
type runsAt map[string]string
@@ -154,16 +111,10 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
}
// Alone, mesh-sdk's tests read the catalogue's trunk, and the verdict names it.
b = newBench(t)
const contracts = "novox/mesh-contracts"
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2, contracts: trunkA, DeliveryRepository: trunkA2})
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2})
if out := b.judge(t, caseOf(t, pullOf(SDKRepository, 7, head1))); out.Beside[catalogue] != trunkA {
t.Errorf("the siblings named for the verdict alone = %v, want the catalogue at its trunk %s", out.Beside, trunkA)
}
// mesh-sdk's tests compare its bootstrapped seats with their contracts files, read beside it: mesh-contracts and
// mesh-delivery, at their trunks alone (a group try is placed by the same besideFor, so it lays them out the same way).
if got := besideOf(t, b, SDKRepository); got[contracts].Commit != trunkA || got[DeliveryRepository].Commit != trunkA2 {
t.Errorf("beside mesh-sdk alone = %+v, want mesh-contracts at %s and mesh-delivery at %s", got, trunkA, trunkA2)
}
if c := besideOf(t, b, SDKRepository)[catalogue]; c.Commit != trunkA || c.Ref != "" {
t.Errorf("the catalogue beside mesh-sdk alone = %+v, want its trunk %s", c, trunkA)
}
@@ -217,18 +168,6 @@ func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
if h := got[nodeEngineRepo]; h.Commit != hostRuns {
t.Errorf("the node-engine beside a catalogue change = %+v, want the commit the mesh runs %s", h, hostRuns)
}
// A group that holds the node-engine is judged against its head, which is what runs after its merge, never the commit
// the mesh runs now; the verdict names it in no carry (it is a member, not read at a trunk or a running commit).
b = newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, nodeEngineRepo: trunkA2})
b.ex.Runs = runsAt{nodeEngineModule: hostRuns}
out = b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1), pullOf(nodeEngineRepo, 8, head2)))
if h := besideOf(t, b, catalogueRepository)[nodeEngineRepo]; h.Commit != head2 || h.Ref != "refs/pull/8/head" {
t.Errorf("the node-engine beside a catalogue change in a group holding it = %+v, want the member's head %s", h, head2)
}
if out.Beside[nodeEngineRepo] != "" {
t.Errorf("the verdict names the member node-engine in its carry: %v", out.Beside)
}
}
// 1887/A12: a mesh-lab head gets a check: with its trunk known it is judged like any repository, its Go fetch finding
@@ -3,7 +3,6 @@ package check
import (
"encoding/json"
"fmt"
"slices"
"sort"
"strings"
"time"
@@ -48,9 +47,6 @@ type Reader struct {
Runs interface{ CommitOf(module string) string }
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
HearingState StateRead
// BuildState is the builder's builds (mesh-delivery-builder.builds): the builds of a running check's candidate that have
// not ended, which the check waits on before it composes (ADR 0603 §10); without it none is named.
BuildState StateRead
// AskState is the checker's state asks: the asks each running check put (AsksState), which checker_ask_again reads.
AskState StateRead
Now func() time.Time
@@ -164,7 +160,6 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
if err != nil {
return VerdictRead{}, err
}
keys = slices.DeleteFunc(keys, IsTryGroup)
sort.Strings(keys)
keys, cut := capKeys(keys, "")
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
@@ -349,10 +344,6 @@ type Status struct {
// Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by
// group, with that step: nothing will judge them (the lost check of 2026-10-28); checker_ask_again asks one again.
Lost map[string]string `json:"lost"`
// BuildsNotEnded are, of each candidate whose check runs, the builder's builds of its changed modules that are neither
// done nor failed, by group: `<module> <state>`, sorted (no key yet is `not begun`). A check waits on them before it
// composes (ADR 0603 §10), but for the modules its line names as composed without them; builder_build_again builds a failed one again, and the check asks itself again when it is done.
BuildsNotEnded map[string]string `json:"builds_not_ended,omitempty"`
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
// whose ask is given up waits until its bound.
DeadLetters DeadLettersHeard `json:"dead_letters"`
@@ -445,7 +436,6 @@ func (r Reader) Status() (Status, error) {
if err != nil {
return st, err
}
keys = slices.DeleteFunc(keys, IsTryGroup) // a try's candidate is judged by its try, never as a check
sort.Strings(keys)
total := len(keys)
keys, cut := capKeys(keys, "")
@@ -510,12 +500,6 @@ func (r Reader) Status() (Status, error) {
if why := read.lostWhy(c); why != "" {
st.Lost[c.Group] = why
}
if words := r.buildsNotEnded(c); words != "" {
if st.BuildsNotEnded == nil {
st.BuildsNotEnded = map[string]string{}
}
st.BuildsNotEnded[c.Group] = words
}
case IsNodesFailure(v):
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
case IsUntrunked(v):
@@ -679,38 +663,3 @@ func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound
}
return ""
}
// buildsNotEnded are the builder's builds of the candidate's changed modules that are neither done nor failed, as
// Status.BuildsNotEnded says them; empty when none, or when the builder's builds are not readable here.
func (r Reader) buildsNotEnded(c delivery.Candidate) string {
if r.BuildState.Get == nil {
return ""
}
var out []string
for _, p := range c.Pulls {
for module, cm := range p.Fps {
if cm.FP2 == "" || cm.Manifest == "" {
continue
}
key, err := delivery.BuilderBuildKey(module, cm.FP2)
if err != nil {
continue
}
raw, err := r.BuildState.Get(key)
if err != nil {
continue
}
state := "not begun"
if len(raw) > 0 {
b, err := delivery.DecodeBuilderBuild(raw)
if err != nil || b.State == delivery.BuildDone || b.State == delivery.BuildFailed {
continue
}
state = b.State
}
out = append(out, module+" "+state)
}
}
sort.Strings(out)
return strings.Join(slices.Compact(out), ", ")
}
@@ -147,24 +147,10 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed})
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
ckey, _ := delivery.CandidatesKey("feat/x")
// ADR 0603 §10: feat/y's check runs and waits on the builds of its changed modules: web building, mail not begun; api
// is done and named nowhere.
fp, man := "src2:"+strings.Repeat("f", 64), strings.Repeat("e", 64)
py := pullOf(repoA, 8, head2)
py.Modules, py.Fps = []string{"api", "mail", "web"}, map[string]delivery.CandidateModule{"web": {FP2: fp, Manifest: man}, "mail": {FP2: fp, Manifest: man}, "api": {FP2: fp, Manifest: man}}
candY := candidateOf(t, "feat/y", py)
ykey, _ := delivery.CandidatesKey("feat/y")
runningV, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/y", Heads: "0123456789ab", State: delivery.VerdictRunning})
builds := fakeState{}
for module, state := range map[string]string{"web": delivery.BuildBuilding, "api": delivery.BuildDone} {
k, _ := delivery.BuilderBuildKey(module, fp)
builds[k], _ = json.Marshal(delivery.BuilderBuild{V: 1, State: state})
}
rd := Reader{
StepStates: map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()},
CandidateState: fakeState{ckey: cand, ykey: candY}.read(),
VerdictState: fakeState{keyOf(t, "feat/y", candY): runningV}.read(),
BuildState: builds.read(),
CandidateState: fakeState{ckey: cand}.read(),
VerdictState: fakeState{}.read(),
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
HearingState: fakeState{"dead-letters": []byte(`{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],` +
`"why":"DEAD_LETTERS is not heard, so an ask a build seat gives up on is waited for until its bound: SEAT_NODE_BUILD_GO: permissions violation",` +
@@ -181,9 +167,6 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
t.Errorf("unjudged = %v", st.Unjudged)
}
if got := st.BuildsNotEnded; len(got) != 1 || got["feat/y"] != "mail not begun, web building" {
t.Errorf("builds not ended = %v, want feat/y's mail not begun and web building", got)
}
// Issue 734, review round 1: a checker that does not hear DEAD_LETTERS says so in its status, with why.
if st.DeadLetters.Heard || !strings.Contains(st.DeadLetters.Why, "permissions violation") {
t.Errorf("dead letters = %+v, want not heard, saying why", st.DeadLetters)
@@ -14,7 +14,6 @@ import (
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -28,8 +27,6 @@ const (
TryTool = "checker_try"
TryReadTool = "checker_try_read"
TryEndTool = "checker_try_end"
// TriesTool lists the tries that wait or run (OpenTries).
TriesTool = "checker_tries"
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
TriesState = "tries"
TriesBucket = delivery.CheckerModule + "_" + TriesState
@@ -65,28 +62,18 @@ var (
// Try is the value of a tries key.
type Try struct {
V int `json:"v"`
ID string `json:"id"`
// Repository, Head, Trunk and Branch are the first member's (the try's headline, and all a try of one repository
// names); Members are every member of the try, by repository: each an exact commit merged onto a trunk commit (a
// group try, issue 1921). A try written before members read as one member.
V int `json:"v"`
ID string `json:"id"`
Repository string `json:"repository"`
// Head is the commit tried; Trunk the trunk commit it is merged onto, and Branch the trunk's branch when it was asked by name.
Head string `json:"head"`
Trunk string `json:"trunk"`
Branch string `json:"branch,omitempty"`
Members []TryMember `json:"members,omitempty"`
Head string `json:"head"`
Trunk string `json:"trunk"`
Branch string `json:"branch,omitempty"`
// Tests is a go test -run pattern: only those tests run, and nothing is kept. Level is unit or integration: only that
// level runs. Empty is every test of both levels.
Tests string `json:"tests,omitempty"`
Level string `json:"level,omitempty"`
State string `json:"state"`
// Told is true from the moment a whole try's start is said on the checker's stream until its end is said there: kept in the
// try's own state, so an end that is written and not said (the checker stopped between) is said when the checker starts.
Told bool `json:"told,omitempty"`
// Waiting is what the running try waits for now (its group's candidate from the indexer, ADR 0639 §5), empty when it waits for nothing;
// try-read gives it, so a try that sits still says why.
Waiting string `json:"waiting,omitempty"`
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
Said string `json:"said,omitempty"`
// Took is the time from its start to its end, as Go writes a duration.
@@ -96,11 +83,6 @@ type Try struct {
Failed []string `json:"failed,omitempty"`
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
Machines []string `json:"machines,omitempty"`
// Composition is what the composition on every placed node came to (a whole try, not one narrowed to a level or to
// some tests), and Notes the words the check noted on its way (the manifests judged, the module graph, the kept
// verdicts taken): the same as a group's pull-request check reports.
Composition *delivery.Composition `json:"composition,omitempty"`
Notes []string `json:"notes,omitempty"`
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
Asks []string `json:"asks,omitempty"`
@@ -112,23 +94,6 @@ type Try struct {
Ended time.Time `json:"ended,omitzero"`
}
// TryMember is one repository of a try: the exact commit asked, never the branch's head now, and the trunk commit it is
// merged onto. Branch is the trunk's branch when the trunk was asked by name.
type TryMember struct {
Repository string `json:"repository"`
Head string `json:"head"`
Trunk string `json:"trunk"`
Branch string `json:"branch,omitempty"`
}
// MembersOf are the try's members: its Members, or the one its headline names.
func (t Try) MembersOf() []TryMember {
if len(t.Members) > 0 {
return t.Members
}
return []TryMember{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk, Branch: t.Branch}}
}
// TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream.
const TryAsked = "asked"
@@ -183,17 +148,6 @@ func (w *tryWriter) heard(askID string, step *delivery.Step) {
}
}
// waiting writes what the try waits for now ("" when it waits for nothing).
func (w *tryWriter) waiting(why string) {
w.mu.Lock()
defer w.mu.Unlock()
if w.stopped || w.t.Waiting == why {
return
}
w.t.Waiting = why
w.write()
}
// write writes the record at the revision written last; a refusal stops the writes.
func (w *tryWriter) write() {
data, err := encodeTry(w.t)
@@ -241,7 +195,6 @@ func DecodeTry(raw []byte) (Try, error) {
if !tryKeyOK.MatchString(t.ID) || t.Repository == "" || t.Head == "" || t.Trunk == "" {
return Try{}, errors.New("not a try: it names no id, repository, head or trunk")
}
t.Members = t.MembersOf()
return t, nil
}
@@ -253,14 +206,6 @@ func encodeTry(t Try) ([]byte, error) {
// TryAsk is what checker_try is given.
type TryAsk struct {
Repository, Head, Trunk, Tests, Level string
// Members are the repositories of a group try, each an exact commit and, optionally, its trunk (a commit, or the
// trunk's branch for its head now); the single Repository, Head and Trunk above are one member. Both at once are refused.
Members []TryAskMember
}
// TryAskMember is one repository of a group try as it is asked.
type TryAskMember struct {
Repository, Head, Trunk string
}
// TryStore is the tries state as the tools see it.
@@ -276,16 +221,21 @@ func NotChecked(repository string) error {
return fmt.Errorf("no level of %s is checked by the checker", repository)
}
// tryMember checks one member as it is asked and resolves its trunk: the gate's trunk head of its repository, a commit
// asked, or the trunk's branch (its head now). A repository with no trunk head is no repository the checker checks.
func tryMember(trunks map[string]string, a TryAskMember) (TryMember, error) {
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the
// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
head := strings.ToLower(strings.TrimSpace(a.Head))
switch {
case !tryRepoOK.MatchString(repo) || strings.Contains(repo, ".."):
return TryMember{}, fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
return "", fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
case !tryHeadOK.MatchString(head):
return TryMember{}, fmt.Errorf("the head %q of %s is not a full commit, 40 or 64 hex digits", a.Head, repo)
return "", fmt.Errorf("the head %q is not a full commit, 40 or 64 hex digits", a.Head)
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
}
trunkHead := ""
for r, h := range trunks {
@@ -294,67 +244,20 @@ func tryMember(trunks map[string]string, a TryAskMember) (TryMember, error) {
}
}
if trunkHead == "" {
return TryMember{}, NotChecked(repo)
return "", NotChecked(repo)
}
m := TryMember{Repository: repo, Head: head}
t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
switch trunk := strings.TrimSpace(a.Trunk); {
case trunk == "":
m.Trunk = trunkHead
t.Trunk = trunkHead
case tryHeadOK.MatchString(strings.ToLower(trunk)):
m.Trunk = strings.ToLower(trunk)
t.Trunk = strings.ToLower(trunk)
case tryTrunkOK.MatchString(trunk) && !strings.Contains(trunk, ".."):
// The checker judges pull requests into the trunk alone (ADR 0484 clause 11): a branch named is the trunk's head now.
m.Trunk, m.Branch = trunkHead, trunk
t.Trunk, t.Branch = trunkHead, trunk
default:
return TryMember{}, fmt.Errorf("the trunk %q of %s is neither a full commit nor a branch", a.Trunk, repo)
return "", fmt.Errorf("the trunk %q is neither a full commit nor a branch", a.Trunk)
}
return m, nil
}
// sameMembers says whether two tries ask the same commits onto the same trunk commits.
func sameMembers(a, b []TryMember) bool {
return slices.EqualFunc(a, b, func(x, y TryMember) bool {
return x.Repository == y.Repository && x.Head == y.Head && x.Trunk == y.Trunk
})
}
// AskTry writes a new try, queued, and answers its id: a try of the same commits that waits or runs is the answer to a second
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. A try is a group of
// one or more members (issue 1921), each an exact commit, never the branch's head now: a push after the ask changes nothing
// of what is tried. trunks are the gate's trunk heads of every repository (source.main): a repository with none is no
// repository the checker checks. A trunk asked by name is its trunk head now.
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
asked := a.Members
if a.Repository != "" || a.Head != "" || a.Trunk != "" {
if len(asked) > 0 {
return "", errors.New("name the members, or one repository with its head, not both")
}
asked = []TryAskMember{{Repository: a.Repository, Head: a.Head, Trunk: a.Trunk}}
}
if len(asked) == 0 {
return "", errors.New("name what to try: members (repository and head each), or a repository and its head")
}
switch {
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
}
var members []TryMember
for _, m := range asked {
tm, err := tryMember(trunks, m)
if err != nil {
return "", err
}
if slices.ContainsFunc(members, func(o TryMember) bool { return o.Repository == tm.Repository }) {
return "", fmt.Errorf("%s is named twice: a try holds one commit of a repository", tm.Repository)
}
members = append(members, tm)
}
slices.SortFunc(members, func(x, y TryMember) int { return strings.Compare(x.Repository, y.Repository) })
first := members[0]
t := Try{Repository: first.Repository, Head: first.Head, Trunk: first.Trunk, Branch: first.Branch, Members: members,
Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
keys, err := store.Keys()
if err != nil {
return "", fmt.Errorf("the tries cannot be listed: %w", err)
@@ -388,10 +291,10 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
if !live[i] {
continue
}
if sameMembers(cur.MembersOf(), t.Members) && cur.Tests == t.Tests && cur.Level == t.Level {
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
return cur.ID, nil
}
said := fmt.Sprintf("%s (%s, %s)", cur.ID, sayMembers(cur.MembersOf()), cur.State)
said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)
if cur.HoldsANode() {
holding = append(holding, said)
} else {
@@ -476,50 +379,12 @@ func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
return t, nil
}
// IsTryGroup says a candidates key is the group of a try (the indexer makes the candidate of a group try's previews, for the
// manifest validator and the composition): the try judges it, so the checker's check of candidates leaves it alone, writes
// no verdict and owes the forge no status for it.
func IsTryGroup(key string) bool { return tryKeyOK.MatchString(key) }
// TryOptions narrow what a try runs; a pull request's check has none.
type TryOptions struct {
Level string
Run string
}
// Whole says the try runs everything a group's pull-request check runs (both levels of every member, the manifests judged by
// the manifest validator, the module graph, the composition on every placed node): it is narrowed to neither a level nor
// some tests. Such a try judges by the same code as the check, so the two cannot drift (issue 1921).
func (o *TryOptions) Whole() bool { return o != nil && o.Level == "" && o.Run == "" }
// TryCase is the case a try is judged as: one pull of the group per member at its exact commit, with no pull request number
// (no ref: the commit is fetched as it is), the group named by the try's key, and the heads hash its candidate is keyed by.
func TryCase(key string, t Try, watch func(string, *delivery.Step)) *Case {
cs := &Case{Key: key, Group: key, Watch: watch, Trunk: map[string]string{}, Queued: t.Asked, Progress: &Progress{},
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
var heads []delivery.Head
for _, m := range t.MembersOf() {
cs.Pulls = append(cs.Pulls, Pull{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
cs.Trunk[m.Repository] = m.Trunk
heads = append(heads, delivery.Head{Repository: m.Repository, Head: m.Head})
}
cs.HeadsHash, _ = delivery.HeadsHash(heads)
cs.Heads = cs.HeadsHash
return cs
}
// compositionMade says whether a composition was made; notMadeWords what it says when it was not.
func compositionMade(c *delivery.Composition) bool {
return c != nil && c.State != delivery.CompositionNotRun
}
func notMadeWords(c *delivery.Composition) string {
if c == nil || c.Words == "" {
return "no reason was given"
}
return c.Words
}
// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check (a check ask, ADR
// 0525 §1), and writes its end. It holds no timer: a try starts when its key is written.
type Tryer struct {
@@ -531,13 +396,7 @@ type Tryer struct {
Now func() time.Time
Logf func(format string, args ...any)
// Say publishes an event of the checker's stream (tryevent.Started, tryevent.Ended) with its body; nil says nothing. The
// indexer, which hears them, writes and deletes the candidate of a whole try's group.
// It answers the publish's error, nil only when the stream acknowledged it: an end not acknowledged is said again.
Say func(ctx context.Context, event string, body tryevent.Event) error
mu sync.Mutex
ending map[string]bool // tries whose end is being said now
wg sync.WaitGroup
inflight map[string]context.CancelFunc
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
@@ -577,7 +436,6 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
stop()
}
tr.mu.Unlock()
tr.sendEnd(ctx, key) // ended by hand while it ran, or its end was written and not said: the indexer drops its candidate
return
}
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
@@ -633,7 +491,6 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return
}
t.State, t.Started = TryRunning, tr.now().UTC()
t.Told = (&TryOptions{Level: t.Level, Run: t.Tests}).Whole() // its start is said below; its end is owed from now on
data, err := encodeTry(t)
if err != nil {
return
@@ -645,15 +502,12 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return
}
progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev}
if t.Told {
started := tryevent.Event{ID: key}
for _, m := range t.MembersOf() {
started.Members = append(started.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
}
tr.say(ctx, tryevent.Started, started)
}
cs := TryCase(key, t, progress.heard)
cs.Wait = progress.waiting
cs := &Case{Key: key, Group: key, Watch: progress.heard,
Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}},
Trunk: map[string]string{t.Repository: t.Trunk},
Queued: t.Asked,
Progress: &Progress{},
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
judging := ctx
if tr.NotTaken > 0 {
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
@@ -663,18 +517,12 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
}
t, rev = progress.end()
t.Waiting = ""
v := out.Verdict
t.Ended = tr.now().UTC()
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
t.Composition = v.Composition
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
if len(cs.Pulls) > 1 {
t.Said = sayMembers(t.MembersOf()) + ": " + t.Said
}
t.Notes = cs.Notes()
for _, n := range t.Notes {
for _, n := range cs.Notes() {
t.Said += " | " + n
}
switch {
@@ -684,16 +532,6 @@ func (tr *Tryer) run(ctx context.Context, key string) {
if IsLostAsk(v) || IsNotTaken(v) {
t.Failed = []string{v.Failed.Words}
}
case v.State == delivery.VerdictPass && cs.Try.Whole() && compositionMade(v.Composition) && strings.Contains(v.Composition.Words, delivery.ComposeNoAnswer):
// A node that gave no answer passes a group's check with a warning; a try that is to say the group composes says it
// did not, naming the node (ADR 0637 §3).
t.State = TryError
t.Failed = []string{"the composition was not made on every placed node: " + v.Composition.Words}
case v.State == delivery.VerdictPass && cs.Try.Whole() && !compositionMade(v.Composition):
// A whole try that judged the tests and not the manifests and the composition has not tried what its group's check
// judges: it is no pass, and it says why (failure must be loud). It is an error, not the change's failure.
t.State = TryError
t.Failed = []string{"the composition was not made: " + notMadeWords(v.Composition)}
case v.State == delivery.VerdictPass:
t.State = TryPass
default:
@@ -711,97 +549,6 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return
}
tr.logf("%s: %s", key, t.State)
tr.sendEnd(ctx, key)
}
// sendEnd says the end of an ended try whose start was said (Told) with the state it ended in, then clears Told in the try's
// own state; a checker that stopped between the two says it again at its start. Every end is said so: pass, fail, error and
// by hand.
func (tr *Tryer) sendEnd(ctx context.Context, key string) {
tr.mu.Lock()
if tr.ending[key] {
tr.mu.Unlock()
return
}
if tr.ending == nil {
tr.ending = map[string]bool{}
}
tr.ending[key] = true
tr.mu.Unlock()
defer func() { tr.mu.Lock(); delete(tr.ending, key); tr.mu.Unlock() }()
raw, rev, found, err := tr.Tries.Get(ctx, key)
if err != nil || !found {
return
}
t, err := DecodeTry(raw)
if err != nil || !t.Over() || !t.Told {
return
}
if tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) != nil {
return // not acknowledged: Told stays, and the next Put or Reconsider says the end again
}
t.Told = false
data, err := encodeTry(t)
if err == nil {
_, err = tr.Tries.Update(ctx, key, data, rev)
}
if err != nil && !errors.Is(err, ErrConflict) {
tr.logf("%s: its end was said and Told could not be cleared: %v", key, err)
}
}
// say publishes a try event; a failure is said in the log (the indexer finds the open tries again at its start).
func (tr *Tryer) say(ctx context.Context, event string, body tryevent.Event) error {
if tr.Say == nil {
return nil
}
err := tr.Say(ctx, event, body)
if err != nil {
tr.logf("%s of %s could not be said: %v", event, body.ID, err)
}
return err
}
// OpenTries are every try that waits or runs now: what checker_tries answers, so a part that missed an event (the indexer at
// its start) finds them without reading the checker's state. All the keys are read (an old open try is as open as a new one), in
// parallel.
func OpenTries(store TryStore) ([]tryevent.Open, error) {
keys, err := store.Keys()
if err != nil {
return nil, fmt.Errorf("the tries cannot be listed: %w", err)
}
sort.Slice(keys, func(i, j int) bool {
if len(keys[i]) != len(keys[j]) {
return len(keys[i]) > len(keys[j])
}
return keys[i] > keys[j]
})
found := make([]*tryevent.Open, len(keys))
if err := parallel(len(keys), func(i int) error {
raw, err := store.Get(keys[i])
if err != nil || len(raw) == 0 {
return nil
}
t, err := DecodeTry(raw)
if err != nil || t.Over() {
return nil
}
o := tryevent.Open{ID: t.ID, State: t.State, Whole: (&TryOptions{Level: t.Level, Run: t.Tests}).Whole()}
for _, m := range t.MembersOf() {
o.Members = append(o.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
}
found[i] = &o
return nil
}); err != nil {
return nil, err
}
out := []tryevent.Open{} // none open is an empty list, never null: a reader tells it from no answer (issue 1921)
for _, o := range found {
if o != nil {
out = append(out, *o)
}
}
return out, nil
}
// judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A
@@ -832,15 +579,6 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O
return out, true
}
// sayMembers names a try's members, each repository at its short commit.
func sayMembers(ms []TryMember) string {
var out []string
for _, m := range ms {
out = append(out, fmt.Sprintf("%s@%.12s", m.Repository, m.Head))
}
return strings.Join(out, " + ")
}
// orNone joins the tries said, or says none.
func orNone(said []string) string {
if len(said) == 0 {
@@ -3,14 +3,11 @@ package check
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -73,25 +70,6 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR
}
// Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or
// their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts.
// A group try is the same try whatever the order of its members; the commits asked are the commits tried.
group := TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2}, {Repository: repoA, Head: head1}}}
gid, err := AskTry(store, tryTrunks, group, at.Add(time.Second))
if err != nil || gid == id {
t.Fatalf("the group try = %q, %v, want a try of its own", gid, err)
}
group.Members[0], group.Members[1] = group.Members[1], group.Members[0]
if again, err := AskTry(store, tryTrunks, group, at.Add(time.Minute)); err != nil || again != gid {
t.Errorf("the group asked in another order answered %q, %v, want %q", again, err, gid)
}
if g := readTry(t, kv, gid); len(g.Members) != 2 || g.Members[0].Repository != repoA || g.Members[0].Head != head1 || g.Members[1].Trunk != trunkS {
t.Errorf("group try = %+v, want its members by repository at the commits asked", g.Members)
}
if open, err := OpenTries(store); err != nil || len(open) != 2 || !open[0].Whole || len(open[1].Members) != 2 && len(open[0].Members) != 2 {
t.Errorf("open tries = %+v, %v, want the two queued", open, err)
}
if !IsTryGroup(gid) || IsTryGroup("feat/x") || IsTryGroup("fix/1921-try-1") {
t.Errorf("which candidates keys are a try's group: %s is, a branch is not", gid)
}
onNode := func(i int) Try {
return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning,
Steps: []TryStep{{Ask: "a", State: delivery.StepRunning}}}
@@ -134,154 +112,59 @@ func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agr
}
}
for _, bad := range []TryAsk{{Repository: repoA, Head: "abc"}, {Repository: repoA, Head: head1, Level: "e2e"},
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"},
// A group try (issue 1921): members or one repository, never both or none; one commit per repository; every member checked.
{},
{Repository: repoA, Head: head1, Members: []TryAskMember{{Repository: repoSDK, Head: head2}}},
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoA, Head: head2}}},
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoSDK, Head: "main"}}},
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: "novox/no-trunk", Head: head2}}}} {
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"}} {
if _, err := AskTry(store, tryTrunks, bad, time.Now()); err == nil {
t.Errorf("%+v was taken", bad)
}
}
}
// fakeTryCandidates answers the indexer's candidate of a try's group from memory, recording the keys asked.
type fakeTryCandidates struct {
cand delivery.Candidate
err error
asked []string
// during is called while the try waits for the candidate.
during func()
}
func (f *fakeTryCandidates) TryCandidate(_ context.Context, group, heads string, _ time.Duration) (delivery.Candidate, error) {
if f.during != nil {
f.during()
}
f.asked = append(f.asked, group+"."+heads)
return f.cand, f.err
}
// wholeBench makes the bench judge a whole try as the checker does: the composition on every placed node and the
// indexer's candidate of the try's group.
func wholeBench(b *bench, cand delivery.Candidate) (*fakeCompositions, *fakeTryCandidates) {
comps := &fakeCompositions{v: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14", "shanks"}, Took: 40}}
cands := &fakeTryCandidates{cand: cand}
b.ex.Compositions, b.ex.Candidates = comps, cands
return comps, cands
}
// A group try (issue 1921): several repositories each at the exact commit asked, judged by the code of a group's pull-request
// check; the two cannot drift, and a later check of the same heads keeps what the try passed.
func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5, issue 1921
func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5
ctx := context.Background()
kv := newMemKV()
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2, Trunk: trunkS}, {Repository: repoA, Head: head1, Trunk: trunkA}}}, time.Now())
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: trunkA}, time.Now())
if err != nil {
t.Fatal(err)
}
b := newBench(t)
comps, cands := wholeBench(b, delivery.Candidate{FPSet: "fps-of-the-try"})
written := &writtenTries{memKV: kv}
var waiting string
cands.during = func() { waiting = readTry(t, kv, id).Waiting } // ADR 0639 §5: the stored try says what it waits for
var said []string
tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
said = append(said, fmt.Sprintf("%s %s %d %s", event, e.ID, len(e.Members), e.State))
return nil
}}
tr := &Tryer{Tries: written, Judge: b.ex}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
// The indexer hears the start with the members at their commits, and the end (it writes and deletes the group's candidate).
if len(said) != 2 || said[0] != "try.started "+id+" 2 " || said[1] != "try.ended "+id+" 0 "+TryPass {
t.Errorf("events = %v, want the start with 2 members and the end", said)
}
// What the indexer reads: tryevent.Decode accepts the events as said and refuses a body that is no try's.
for _, row := range []struct {
name, event, body string
ok bool
}{
{"a start with its members", tryevent.Started, `{"id":"` + id + `","members":[{"repository":"` + repoA + `","head":"` + head1 + `","trunk":"` + trunkA + `"}]}`, true},
{"an end with its state", tryevent.Ended, `{"id":"` + id + `","state":"pass"}`, true},
{"an id that is no try's", tryevent.Ended, `{"id":"pull-7","state":"pass"}`, false},
{"a start with no member", tryevent.Started, `{"id":"` + id + `"}`, false},
{"a body that is no JSON", tryevent.Ended, `try-7`, false},
} {
if e, err := tryevent.Decode(row.event, []byte(row.body)); (err == nil) != row.ok || (row.ok && e.ID != id) {
t.Errorf("Decode of %s = %+v, %v, want ok=%v", row.name, e, err, row.ok)
}
}
got := readTry(t, kv, id)
if !strings.Contains(waiting, "candidate") || got.Waiting != "" {
t.Errorf("waiting while the candidate was awaited = %q, after the end %q: want the reason, then none", waiting, got.Waiting)
}
if got.Told { // the end was said: nothing is owed
t.Errorf("the ended try still owes its end: %+v", got)
}
if open, err := OpenTries(memTries{kv}); err != nil || len(open) != 0 {
t.Errorf("open tries after the end = %v, %v", open, err)
} else if raw, _ := json.Marshal(open); string(raw) != "[]" { // issue 1921: none open is [] on the wire, never null
t.Errorf("none open is answered %s, want []", raw)
}
if got.State != TryPass || got.Took == "" || len(got.Members) != 2 || got.Members[0].Repository != repoA || got.Members[1].Head != head2 ||
got.Said == "" || got.Levels[delivery.LevelUnit].Run+got.Levels[delivery.LevelUnit].Kept != 4 { // the bench gives both members the same digests
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
t.Fatalf("try = %+v", got)
}
if got.Composition == nil || got.Composition.State != delivery.CompositionPass || len(got.Machines) == 0 {
t.Errorf("the try's composition = %+v, want the composition on every placed node, as a check's", got.Composition)
// Issue 1890: while it runs, the try names its ask as soon as it is put, and then where its step stands; the end
// keeps each ask's node and state.
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked ||
written.running[2].Steps[0].Node != "shanks" {
t.Errorf("the running try's writes = %+v, want its ask named while it ran, then its node", written.running)
}
// The same key the indexer's candidate of the group has: the try's id and the heads of its members.
if len(comps.asked) != 1 || comps.asked[0] != id || len(cands.asked) != 1 || !strings.HasPrefix(cands.asked[0], id+".") {
t.Errorf("composed %v, candidate asked %v, want the try's key %s", comps.asked, cands.asked, id)
if len(got.Steps) != 1 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed {
t.Errorf("the ended try's steps = %+v", got.Steps)
}
// Issue 1890: while it runs, the try names its asks as soon as they are put, and then where each step stands.
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked {
t.Errorf("the running try's writes = %+v, want its ask named while it ran", written.running)
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
if len(got.Asks) != 1 {
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
}
if len(got.Steps) != 2 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed || len(got.Asks) != 2 {
t.Errorf("the ended try's steps = %+v, asks %v", got.Steps, got.Asks)
}
for _, a := range got.Asks { // the delivery view counts their steps as a try's (build_usage)
if _, err := delivery.AskStepKey("shanks", a); err != nil {
t.Errorf("the try names %q, which is no ask id: %v", a, err)
}
}
// Each member is fetched at the exact commit asked, merged onto its own trunk, with no pull request's ref.
fetch := b.steps.tool("fetch")
if len(fetch) != 2 {
t.Fatalf("fetch = %+v, want one per member", fetch)
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
}
for _, f := range fetch {
want := map[string][2]string{repoA: {head1, trunkA}, repoSDK: {head2, trunkS}}[f.args["repository"].(string)]
if f.args["commit"] != want[0] || f.args["base"] != want[1] || f.args["ref"] != "" {
t.Errorf("fetch = %+v, want the exact commit %s merged onto %s", f.args, want[0], want[1])
}
}
// A whole try writes the merge previews the indexer makes its candidate from; it writes no status and asks no Approve.
if keys, _ := b.prev.Keys(ctx); len(keys) != 2 {
t.Errorf("a whole try wrote the previews %v, want one per member", keys)
if keys, _ := b.prev.Keys(ctx); len(keys) != 0 {
t.Errorf("a try wrote the previews %v", keys)
}
// An ended try is not run again.
raw, _, _, _ = kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if n := len(b.steps.tool("fetch")); n != 2 {
if n := len(b.steps.tool("fetch")); n != 1 {
t.Errorf("an ended try was fetched again: %d fetches", n)
}
// The kept verdicts: what the try passed is kept under the digests of the merged trees, so the pull-request check of
// the same head on the same trunk runs none of it again (and a push of other code runs only what changed).
before := len(b.steps.tool("test"))
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if out.Verdict.State != delivery.VerdictPass || out.Verdict.Levels[delivery.LevelUnit].Kept == 0 || out.Verdict.Levels[delivery.LevelUnit].Run != 0 ||
out.Verdict.Levels[delivery.LevelIntegration].Run != 0 {
t.Errorf("the check of the tried head = %+v, want everything the try passed kept and nothing run", out.Verdict.Levels)
}
if after := len(b.steps.tool("test")); after != before {
t.Errorf("the check of the tried head ran %d test steps, want none: the try's passes are kept", after-before)
}
}
func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
@@ -292,16 +175,10 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
t.Fatal(err)
}
b := newBench(t)
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(context.Context, string, tryevent.Event) error {
t.Error("a try of one level and some tests said an event: it has no candidate")
return nil
}}
tr := &Tryer{Tries: kv, Judge: b.ex}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if open, _ := OpenTries(memTries{kv}); len(open) != 0 {
t.Errorf("open = %v", open)
}
tests := b.steps.tool("test")
if len(tests) != 1 || tests[0].args["level"] != delivery.LevelIntegration || tests[0].args["run"] != "TestBus|TestStore" {
t.Fatalf("tests = %+v, want the integration level alone with the run pattern", tests)
@@ -315,24 +192,6 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
if got := readTry(t, kv, id); got.State != TryPass || !strings.Contains(got.Said, "only the tests TestBus|TestStore") {
t.Errorf("try = %+v", got)
}
t.Run("a try of one level keeps only that level", func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: delivery.LevelUnit}, time.Now())
b := newBench(t)
tr := &Tryer{Tries: kv, Judge: b.ex}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
keys, _ := b.kept.Keys(ctx)
if got := readTry(t, kv, id); got.State != TryPass || len(keys) == 0 {
t.Fatalf("try = %+v, kept %v", got, keys)
}
for _, k := range keys {
if !strings.HasPrefix(k, delivery.LevelUnit+".") {
t.Errorf("a try of the unit level kept %s", k)
}
}
})
}
func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) {
@@ -349,135 +208,26 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T)
!strings.Contains(got.Failed[1], "integration: fixture/failing/bad.TestThreeIsFour") { // both levels ran (the operator, 2026-11-02)
t.Errorf("try = %+v", got)
}
// A whole try that could not judge the manifests and the composition is no pass; a failed composition is named beside
// the failed tests (the group's check does as much), and a candidate of other heads is not the try's.
for _, row := range []struct {
name string
cand *fakeTryCandidates
comp delivery.CompositionVerdict
state, said string
}{
{name: "the indexer wrote no candidate", cand: &fakeTryCandidates{err: errors.New("the indexer wrote no candidate within 5m0s")}, state: TryError, said: "the indexer wrote no candidate within 5m0s"},
{name: "a node that gave no composition answer", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x", Warnings: []string{"g14: " + delivery.ComposeNoAnswer}},
state: TryError, said: "g14: " + delivery.ComposeNoAnswer},
{name: "a composition refused on a node", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "not composed on g14: web: needs a state mail does not keep"},
state: TryFail, said: "not composed on g14: web: needs a state mail does not keep"},
} {
kv := newMemKV()
b := newBench(t)
comps, _ := wholeBench(b, delivery.Candidate{})
b.ex.Candidates, comps.v = row.cand, row.comp
if row.comp.State == "" {
comps.v.State = delivery.CompositionPass
}
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
var ends []string
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
if event == tryevent.Ended {
ends = append(ends, e.State)
}
return nil
}}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if len(ends) != 1 || ends[0] != row.state { // every end says its state: error as well as fail
t.Errorf("%s: ends said = %v, want once %s", row.name, ends, row.state)
}
if got := readTry(t, kv, id); got.State != row.state || !strings.Contains(strings.Join(got.Failed, " | ")+got.Said, row.said) {
t.Errorf("%s: try = %s, failed %v, said %q, want %s naming %q", row.name, got.State, got.Failed, got.Said, row.state, row.said)
}
}
// No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4).
}
func TestATryLeftRunningIsTakenOver(t *testing.T) {
ctx := context.Background()
// Whether the start was said is in the try's own state (Told), so a checker that starts later says the end of a try it did
// not start, whatever way the try ended.
for _, row := range []struct {
name string
told bool
failFirst bool // the stream does not acknowledge the first end said
after func(t *testing.T, kv *memKV, id string)
ends []string
}{
{name: "the first publish of the end is not acknowledged", told: true, failFirst: true, ends: []string{TryPass}},
{name: "left running after its start was said", told: true, ends: []string{TryPass}},
{name: "left running with no start said", told: false, ends: []string{TryPass}},
{name: "ended by hand after the start was said", told: true, ends: []string{TryError}, after: func(t *testing.T, kv *memKV, id string) {
if _, err := EndTry(endable{kv}, id, "stuck", time.Now()); err != nil {
t.Fatal(err)
}
}},
} {
t.Run(row.name, func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
// Left running by a checker that stopped.
raw, rev, _, _ := kv.Get(ctx, id)
cur, _ := DecodeTry(raw)
cur.State, cur.Told = TryRunning, row.told
data, _ := encodeTry(cur)
if _, err := kv.Update(ctx, id, data, rev); err != nil {
t.Fatal(err)
}
if row.after != nil {
row.after(t, kv, id)
}
b := newBench(t)
wholeBench(b, delivery.Candidate{})
var ends []string
failed := false
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
if event == tryevent.Ended {
if row.failFirst && !failed {
failed = true
return errors.New("the stream did not acknowledge")
}
ends = append(ends, e.State)
}
return nil
}}
tr.Reconsider(ctx)
tr.Wait()
if row.failFirst && !readTry(t, kv, id).Told {
t.Errorf("an end the stream did not acknowledge cleared Told: it is never said again")
}
tr.Reconsider(ctx) // said once: the second look finds nothing owed
tr.Wait()
if row.after == nil {
if got := readTry(t, kv, id); got.State != TryPass {
t.Errorf("the try left running = %+v, want taken over and passed", got)
}
}
if !slices.Equal(ends, row.ends) || readTry(t, kv, id).Told {
t.Errorf("ends said = %v, want %v, and nothing owed after", ends, row.ends)
}
})
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
// Left running by a checker that stopped.
raw, rev, _, _ := kv.Get(ctx, id)
cur, _ := DecodeTry(raw)
cur.State = TryRunning
data, _ := encodeTry(cur)
if _, err := kv.Update(ctx, id, data, rev); err != nil {
t.Fatal(err)
}
b := newBench(t)
tr := &Tryer{Tries: kv, Judge: b.ex}
tr.Reconsider(ctx)
tr.Wait()
if got := readTry(t, kv, id); got.State != TryPass {
t.Errorf("the try left running = %+v, want taken over and passed", got)
}
// An ended try is kept TryKeep and removed by the next Reconsider after.
t.Run("an ended try is kept seven days", func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
raw, rev, _, _ := kv.Get(ctx, id)
cur, _ := DecodeTry(raw)
ended := time.Now()
cur.State, cur.Ended = TryPass, ended
data, _ := encodeTry(cur)
if _, err := kv.Update(ctx, id, data, rev); err != nil {
t.Fatal(err)
}
now := ended.Add(TryKeep - time.Hour)
tr := &Tryer{Tries: kv, Now: func() time.Time { return now }}
tr.Reconsider(ctx)
if _, _, found, _ := kv.Get(ctx, id); !found {
t.Fatal("the try was removed before TryKeep")
}
now = ended.Add(TryKeep + time.Hour)
tr.Reconsider(ctx)
if _, _, found, _ := kv.Get(ctx, id); found {
t.Error("the try was kept past TryKeep")
}
})
}
+7 -36
View File
@@ -130,40 +130,14 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
if m.SecretsOwner != "mesh-delivery-checker" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3). A whole try says try.started with its members and try.ended, so the indexer writes and deletes the candidate of its group (issue 1921).
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}}; !reflect.DeepEqual(m.Streams, want) {
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3).
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**"}}}; !reflect.DeepEqual(m.Streams, want) {
t.Errorf("streams = %+v, want %+v", m.Streams, want)
}
// Only the checker's account publishes try.* on its stream checks: the stream is declared by its owner alone, and no other
// module of the repository declares a stream carrying a try event (the indexer and the others only consume them).
siblings, err := filepath.Glob("../*/module.json")
if err != nil || len(siblings) < 2 {
t.Fatalf("the sibling manifests = %v, %v", siblings, err)
}
for _, f := range siblings {
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
var other struct {
Module string `json:"module"`
Streams []stream `json:"streams"`
}
if err := json.Unmarshal(raw, &other); err != nil {
t.Fatalf("%s: %v", f, err)
}
for _, st := range other.Streams {
for _, e := range st.Events {
if strings.HasPrefix(e, "try.") && other.Module != m.Module {
t.Errorf("%s declares the stream %s with %s: only the checker's account publishes try events", other.Module, st.Name, e)
}
}
}
}
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check
// is asked again by hand (checker_ask_again).
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code.
@@ -200,16 +174,13 @@ func TestManifestProcess(t *testing.T) { // 1678/AC1
if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_CHECKER_SETTINGS"] != "${dir:state}/settings.json" {
t.Errorf("env = %v, want the broker file and the settings file", p["env"])
}
// Its settings hold its answer bound (ADR 0593 §7: compose-answer-bound, 60 s, the checker's) and its wait for the
// candidate's builds (ADR 0603 §10: build-wait-bound, 1800 s), never a capacity of its own (ADR 0525 §4).
// Its settings hold its answer bound alone (ADR 0593 §7: compose-answer-bound, 60 s, the checker's), never a capacity
// of its own (ADR 0525 §4).
for _, r := range m.Resources {
if r["id"] == "settings" && !reflect.DeepEqual(r["takes-config"], []any{"compose-answer-bound", "build-wait-bound"}) {
t.Errorf("the settings resource %v takes other than compose-answer-bound and build-wait-bound", r)
if r["id"] == "settings" && !reflect.DeepEqual(r["takes-config"], []any{"compose-answer-bound"}) {
t.Errorf("the settings resource %v takes more than compose-answer-bound", r)
}
}
if c, ok := m.Config["build-wait-bound"]; !ok || c.Default != float64(1800) || c.Trusted {
t.Errorf("build-wait-bound = %+v, want 1800 by default, not trusted", c)
}
if c, ok := m.Config["compose-answer-bound"]; !ok || c.Default != float64(60) || c.Trusted {
t.Errorf("compose-answer-bound = %+v, want 60 by default, not trusted", c)
}
+2 -12
View File
@@ -18,7 +18,7 @@
"mesh-build-node.toolchain.reported"
],
"streams": [
{ "name": "checks", "events": ["preview.**", "owed.**", "try.**"] }
{ "name": "checks", "events": ["preview.**", "owed.**"] }
],
"state": [
{ "name": "asks", "history": 1 },
@@ -56,7 +56,6 @@
"checker_previews",
"checker_pull",
"checker_status",
"checker_tries",
"checker_try",
"checker_try_end",
"checker_try_read",
@@ -82,14 +81,6 @@
"maximum": 600,
"trusted": false,
"default": 60
},
"build-wait-bound": {
"type": "number",
"description": "seconds a check waits for the builder's builds of its candidate to end before it composes with them (novox/hq ADR 0603 §10); a build failed fails the check, naming it, and a build not ended by then is named in the verdict and the check composes without it.",
"minimum": 60,
"maximum": 7200,
"trusted": false,
"default": 1800
}
},
"resources": [
@@ -115,8 +106,7 @@
"mode": "0600",
"owner": "mesh-delivery-checker",
"takes-config": [
"compose-answer-bound",
"build-wait-bound"
"compose-answer-bound"
],
"merge": "json",
"template": "files/settings.json"
@@ -1,266 +0,0 @@
// Command mesh-delivery-forge-bridge-tools serves the forge bridge's tools (novox/hq ADR 0633 §4, issue 1585): the runtime
// launches it for a call. bridge_groups reads the delivery groups the gate holds and what the bridge kept of their ends;
// bridge_group_end_again writes an ask into the bridge's state end-asks and waits for the bridge's answer there, since the
// bridge is the other program and serves no request subject (ADR 0437 §4).
package main
import (
"context"
"encoding/json"
"fmt"
"maps"
"os"
"slices"
"strconv"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"mesh-delivery-forge-bridge/internal/bridge"
"mesh-delivery-forge-bridge/internal/groups"
)
// Tool names.
const (
GroupsTool = "bridge_groups"
EndAgainTool = "bridge_group_end_again"
// MergedTool answers the merged message of a merge on request (issue 1585).
MergedTool = "bridge_merged_message"
)
// asker writes an end-asks value and waits for the bridge's answer to it.
type asker func(key string, value []byte) (groups.EndAsk, error)
// Tools are the bridge's tools: groupsRead reads the groups the gate holds with what the bridge kept of them.
func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string) (json.RawMessage, error)) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: MergedTool,
Description: "The merged message of one merge, asked again (issue 1585): the message the bridge wrote on its stream FORGE when the " +
"pull request was merged into the trunk, built the same way from the forge (the merge commit's trees and the manifests " +
"and lock files of its module directories) and from the gate's merges state (the pull request's number, head and " +
"group). For the indexer, which asks it when it takes in a held merge it does not hold in memory (a restart, a new " +
"build), and the ask-again a lost message lacked. About 0.5 MB for a repository of many modules. Answers {message} " +
"with the forge message whole, or why it cannot: the gate holds no merge with that commit, or the forge did not answer.",
Input: map[string]any{"type": "object", "required": []string{"repository", "merge"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "the repository, owner/name, as the forge names it"},
"merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"}}},
Run: func(args map[string]any) (any, error) {
raw, err := merged(str(args, "repository"), str(args, "merge"))
if err != nil {
return nil, err
}
return map[string]any{"message": raw}, nil
}},
{Name: GroupsTool,
Description: "The delivery groups the gate holds (more than one member), each with its members' merge state and whether each " +
"merged message is written on the forge stream, and whether the group's end (forge.group.merged) is written: ended, " +
"or open with why (gathered, or merged and its end not written yet). An open group with every member merged is " +
"what bridge_group_end_again says the end of. Each group names its last end ask (who, why, when) and the bridge's " +
"answer to it, from the state end-asks. A group's name narrows it to that group; open lists only open ones.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "only this group, by the name the gate keys it"},
"open": map[string]any{"type": "boolean", "description": "only the groups whose end is not written"}}},
Run: func(args map[string]any) (any, error) {
rows, unread, err := groupsRead()
if err != nil {
return nil, err
}
only, _ := args["open"].(bool)
group := str(args, "group")
kept := rows[:0]
for _, r := range rows {
if (group != "" && r.Group != group) || (only && !r.Open) {
continue
}
kept = append(kept, r)
}
return map[string]any{"groups": kept, "unread": unread}, nil
}},
{Name: EndAgainTool,
Description: "Say a delivery group's end (forge.group.merged) again, by hand, for a group whose end the indexer did not take " +
"in (a group bridge_groups names as merged with its end not written, or one the indexer waits for). The bridge " +
"writes it again from the gate's merges, with the time appended to the message id so the stream does not drop it as " +
"a duplicate; the indexer takes an end for a group it already took in without changing anything. Refused while " +
"a member is not merged or its merged message is not written. Answers whether it was written.",
Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the group, by the name the gate keys it"},
"why": map[string]any{"type": "string", "description": "why its end is said again, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := groups.NewEndAsk(str(args, "group"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
a, err := ask(key, value)
if err != nil {
return nil, err
}
return map[string]any{"group": a.Group, "written": a.Written, "answer": a.Answer, "by": a.By, "why": a.Why}, nil
}},
}
}
// MergedOf is the merged message of the merge of repository at commit merge, as the bridge writes it: the pull request's
// number, head and group from the gate's merges (raw, by key), the rest from the forge through trees and files
// (bridge.MergedMessage), and, for a member of a delivery group, the group and its members as the process sets them.
func MergedOf(raws map[string][]byte, repository, merge string, trees bridge.Trees, files bridge.Files) (json.RawMessage, error) {
owner, repo, ok := strings.Cut(repository, "/")
if !ok || owner == "" || repo == "" {
return nil, fmt.Errorf("%q is not owner/name", repository)
}
for _, key := range slices.Sorted(maps.Keys(raws)) {
m, err := delivery.DecodeMerge(raws[key])
if err != nil {
continue
}
at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool {
return delivery.SameRepository(p.Repository, repository) && p.Commit == merge
})
if at < 0 {
continue
}
p := m.Pulls[at]
msg, err := bridge.MergedMessage(owner, repo, p.Number, m.Group, p.Head.Head, merge, trees, files)
if err != nil {
return nil, fmt.Errorf("the merged message of %s at %s cannot be built: %w", repository, merge, err)
}
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
return nil, err
}
if _, approved := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; approved && len(m.Approved) >= 2 {
f.Group, f.Members = m.Group, groups.MembersOf(m)
}
out, err := forgemsg.EncodeForge(f)
return out, err
}
return nil, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, so its pull request is not known", repository, merge)
}
// read is the groups the gate holds with what the bridge kept of them: their ends in performed, and the last ask of
// each in end-asks with the bridge's answer.
func read(merges, performed, asks stdio.KeptState) ([]groups.Row, []string, error) {
keys, err := merges.Keys()
if err != nil {
return nil, nil, fmt.Errorf("the gate's merges cannot be listed: %w", err)
}
raws := map[string][]byte{}
for _, k := range keys {
e, err := merges.Get(k)
if err != nil || e == nil {
continue
}
raws[k] = e.Value
}
pkeys, err := performed.Keys()
if err != nil {
return nil, nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err)
}
published, ended := map[string]bool{}, map[string]uint64{}
for _, k := range pkeys {
switch {
case strings.HasPrefix(k, "published."):
published[k] = true
case strings.HasPrefix(k, "groups."):
if e, err := performed.Get(k); err == nil && e != nil {
rev, _ := strconv.ParseUint(strings.TrimSpace(string(e.Value)), 10, 64)
ended[strings.TrimPrefix(k, "groups.")] = rev
}
}
}
akeys, err := asks.Keys()
if err != nil {
return nil, nil, fmt.Errorf("the bridge's state %s cannot be listed: %w", groups.EndAsksState, err)
}
askRaws := map[string][]byte{}
for _, k := range akeys {
if e, err := asks.Get(k); err == nil && e != nil {
askRaws[k] = e.Value
}
}
rows, unread := groups.Rows(raws, published, ended, askRaws)
return rows, unread, nil
}
// askEnd writes the end-asks value at key and waits, on a watch of the key, at most bound for the bridge's answer.
func askEnd(s stdio.KeptState, bound time.Duration) asker {
return func(key string, value []byte) (groups.EndAsk, error) {
answered := make(chan groups.EndAsk, 1)
var rev uint64
var mu sync.Mutex
take := func(revision uint64, v []byte) {
mu.Lock()
mine := rev
mu.Unlock()
if mine == 0 || revision <= mine {
return
}
if a, err := groups.DecodeEndAsk(v); err == nil && a.Done() {
select {
case answered <- a:
default:
}
}
}
if err := s.Watch(key, func(c stdio.StateChange) error {
if c.Op == "put" {
take(c.Revision, c.Value)
}
return nil
}); err != nil {
return groups.EndAsk{}, fmt.Errorf("watching %s.%s: %w", groups.EndAsksState, key, err)
}
r, err := s.Put(key, json.RawMessage(value))
if err != nil {
return groups.EndAsk{}, fmt.Errorf("the ask could not be written to %s: %w", groups.EndAsksBucket, err)
}
mu.Lock()
rev = r
mu.Unlock()
if e, err := s.Get(key); err == nil && e != nil {
take(e.Revision, e.Value)
}
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
defer cancel()
select {
case a := <-answered:
return a, nil
case <-wait.Done():
return groups.EndAsk{}, fmt.Errorf("the bridge did not answer within %s: its answer will be in %s.%s", bound, groups.EndAsksBucket, key)
}
}
}
// askCaller calls a tool as this bundle's module, through the runtime (stdio.Ask), the module's own invokes granting it.
func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) {
return stdio.Ask(key, body)
}
func main() {
merges, performed, asks := stdio.State(delivery.GateModule+".merges"), stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState)
tools := Tools(askEnd(asks, 25*time.Second), time.Now,
func() ([]groups.Row, []string, error) { return read(merges, performed, asks) },
func(repository, merge string) (json.RawMessage, error) {
keys, err := merges.Keys()
if err != nil {
return nil, fmt.Errorf("the gate's merges cannot be listed: %w", err)
}
raws := map[string][]byte{}
for _, k := range keys {
if e, err := merges.Get(k); err == nil && e != nil {
raws[k] = e.Value
}
}
ctx := context.Background()
return MergedOf(raws, repository, merge, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller))
})
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err)
os.Exit(1)
}
}
@@ -1,190 +0,0 @@
package main
import (
"bytes"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-forge-bridge/internal/bridge"
"mesh-delivery-forge-bridge/internal/groups"
)
// 1585/T18, T19 (ADR 0633 §4): bridge_groups names each group the gate holds with its members' state and whether its end is
// written (open or ended, with why), narrowed by group and by open; bridge_group_end_again writes an ask naming the group,
// who asked and why, and answers what the bridge answered.
func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) {
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
enc := func(group string, state2 delivery.MergeState, c string) []byte {
v, err := delivery.EncodeMerge(delivery.Merge{Group: group, Ask: "ask", Approved: map[string]string{"novox/a@1": h1, "novox/b@2": h2},
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 1, Head: h1}, State: delivery.MergeMerged, Commit: c1},
{Head: delivery.Head{Repository: "novox/b", Number: 2, Head: h2}, State: state2, Commit: c}}})
if err != nil {
t.Fatal(err)
}
return v
}
lone, _ := delivery.EncodeMerge(delivery.Merge{Group: "solo", Ask: "ask", Approved: map[string]string{"novox/a@5": h1},
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 5, Head: h1}, State: delivery.MergeMerged, Commit: c1}}})
merges := map[string][]byte{"ended": enc("ended", delivery.MergeMerged, c2), "ready": enc("ready", delivery.MergeMerged, c2),
"gathered": enc("gathered", delivery.MergeMerging, ""), "solo": lone, "bad": []byte("{")}
performed := map[string]bool{}
for _, c := range []string{c1, c2} {
performed[groups.PublishedKey("novox/a", 1, c)], performed[groups.PublishedKey("novox/b", 2, c)] = true, true
}
// The state end-asks keeps the last ask of the group ready, answered, and one that does not read.
answeredAsk, _ := json.Marshal(groups.EndAsk{V: 1, Group: "ready", Why: "lost", By: "lead@shanks", At: time.Date(2026, 11, 4, 22, 0, 0, 0, time.UTC),
Written: true, Answer: "the end of the group is written again", Answered: time.Date(2026, 11, 4, 22, 0, 1, 0, time.UTC)})
asks := map[string][]byte{"ready": answeredAsk, "gathered": []byte("{")}
var asked groups.EndAsk
var askedKey string
ask := func(key string, value []byte) (groups.EndAsk, error) {
askedKey = key
a, err := groups.DecodeEndAsk(value)
a.Written, a.Answer, a.Answered = true, "the end of the group is written again", time.Now()
asked = a
return a, err
}
tools := Tools(ask, func() time.Time { return time.Date(2026, 11, 4, 23, 0, 0, 0, time.UTC) },
func() ([]groups.Row, []string, error) {
rows, unread := groups.Rows(merges, performed, map[string]uint64{"ended": 7}, asks)
return rows, unread, nil
}, nil)
call := func(name string, args map[string]any) any {
for _, tl := range tools {
if tl.Name == name {
out, err := tl.Run(args)
if err != nil {
t.Fatalf("%s: %v", name, err)
}
return out
}
}
t.Fatalf("no tool %s", name)
return nil
}
rowsOf := func(args map[string]any) map[string]groups.Row {
raw, _ := json.Marshal(call(GroupsTool, args))
var out struct {
Groups []groups.Row `json:"groups"`
Unread []string `json:"unread"`
}
if err := json.Unmarshal(raw, &out); err != nil {
t.Fatal(err)
}
m := map[string]groups.Row{}
for _, r := range out.Groups {
m[r.Group] = r
}
if len(args) == 0 && strings.Join(out.Unread, " ") != "bad end-asks.gathered" {
t.Errorf("unread = %v, want bad and end-asks.gathered", out.Unread)
}
return m
}
all := rowsOf(map[string]any{})
if len(all) != 3 || !all["ended"].Ended || all["ended"].EndedAt != 7 {
t.Fatalf("groups = %+v", all)
}
if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published {
t.Errorf("ready = %+v, want open, merged and published, its end the thing to say", r)
}
if a := all["ready"].LastEndAsk; a == nil || a.By != "lead@shanks" || a.Why != "lost" || !a.Written || a.Answer == "" || all["ended"].LastEndAsk != nil {
t.Errorf("ready's last end ask = %+v, ended's %+v: want ready's ask and its answer, none for ended", a, all["ended"].LastEndAsk)
}
if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != string(delivery.MergeMerging) {
t.Errorf("gathered = %+v", r)
}
if open := rowsOf(map[string]any{"open": true}); len(open) != 2 || open["ended"].Group != "" {
t.Errorf("open = %+v, want ready and gathered", open)
}
if one := rowsOf(map[string]any{"group": "ready"}); len(one) != 1 {
t.Errorf("group ready = %+v", one)
}
out := call(EndAgainTool, map[string]any{"group": "ready", "why": "the indexer never took it in"}).(map[string]any)
if askedKey != "ready" || asked.Group != "ready" || asked.Why != "the indexer never took it in" || out["written"] != true {
t.Errorf("the ask %q %+v, answered %v", askedKey, asked, out)
}
for _, tl := range tools {
if tl.Name == EndAgainTool {
if _, err := tl.Run(map[string]any{}); err == nil {
t.Error("an ask naming no group was taken")
}
}
}
// 1585: bridge_merged_message answers the merged message as the bridge publishes it: for a lone pull request the same bytes
// the event gives through Convert, for a member of a delivery group those with its group and members set as the process
// sets them; a merge the gate does not hold, or a forge that does not answer, is said.
t.Run("bridge_merged_message answers the merged message as published", func(t *testing.T) {
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
trees := func(owner, repo, sha string) (string, map[string]string, error) {
return "root-" + sha[:4], map[string]string{"": "r", "modules/a": "ta-" + sha[:4]}, nil
}
files := func(owner, repo, path, ref string) ([]byte, error) {
if strings.HasSuffix(path, "module.json") {
return []byte(`{"module":"a"}`), nil
}
return nil, bridge.ErrNotFound
}
published := func(repo string, number int, head, branch, merge string) forgemsg.Forge {
body, _ := json.Marshal(map[string]any{"owner": "novox", "repo": repo, "number": number, "head": branch, "head_sha": head, "base": "main", "merge_commit_sha": merge})
msg, err := bridge.Convert(bridge.MergedSubject, body, trees, files)
if err != nil {
t.Fatal(err)
}
f, err := forgemsg.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
return f
}
lone, _ := delivery.EncodeMerge(delivery.Merge{Group: "solo", Ask: "a", Approved: map[string]string{"novox/a@5": h1},
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 5, Head: h1}, State: delivery.MergeMerged, Commit: c1}}})
group, _ := delivery.EncodeMerge(delivery.Merge{Group: "feat/g", Ask: "a", Approved: map[string]string{"novox/a@6": h1, "novox/b@7": h2},
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 6, Head: h1}, State: delivery.MergeMerged, Commit: c1},
{Head: delivery.Head{Repository: "novox/b", Number: 7, Head: h2}, State: delivery.MergeMerging}}})
for _, c := range []struct {
name string
raws map[string][]byte
repository string
merge string
want forgemsg.Forge
fail string
}{
{name: "a lone pull request", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c1, want: published("a", 5, h1, "solo", c1)},
{name: "a member of a group", raws: map[string][]byte{"feat~g": group}, repository: "novox/a", merge: c1, want: func() forgemsg.Forge {
f := published("a", 6, h1, "feat/g", c1)
f.Group, f.Members = "feat/g", []delivery.Head{{Repository: "novox/a", Number: 6, Head: h1}, {Repository: "novox/b", Number: 7, Head: h2}}
return f
}()},
{name: "a merge the gate does not hold", raws: map[string][]byte{"solo": lone, "bad": []byte("{")}, repository: "novox/a", merge: c2, fail: "hold no merge"},
} {
t.Run(c.name, func(t *testing.T) {
raw, err := MergedOf(c.raws, c.repository, c.merge, trees, files)
if c.fail != "" {
if err == nil || !strings.Contains(err.Error(), c.fail) {
t.Fatalf("err %v, want %q", err, c.fail)
}
return
}
want, _ := forgemsg.EncodeForge(c.want)
if err != nil || !bytes.Equal(raw, want) {
t.Fatalf("err %v, answer\n%s\nwant the published message\n%s", err, raw, want)
}
})
}
if _, err := MergedOf(map[string][]byte{"solo": lone}, "novox/a", c1, func(string, string, string) (string, map[string]string, error) {
return "", nil, errors.New("the forge does not answer")
}, files); err == nil || !strings.Contains(err.Error(), "does not answer") {
t.Errorf("a forge that does not answer: %v", err)
}
})
}
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"regexp"
"strings"
"time"
@@ -65,12 +66,58 @@ func callerOn(nc *nats.Conn, user string) caller {
}
}
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner,
// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4).
const commitTreesTool = "gitea.gitea_commit_trees"
// treesVia asks the forge for a commit's trees through call.
func treesVia(ctx context.Context, call caller) bridge.Trees {
return bridge.TreesVia(ctx, bridge.Caller(call))
return func(owner, repo, sha string) (string, map[string]string, error) {
raw, err := call(ctx, commitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha})
if err != nil {
return "", nil, err
}
var a struct {
Root string `json:"root"`
Trees map[string]string `json:"trees"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
}
if a.Root == "" {
return "", nil, fmt.Errorf("the answer of %s names no root tree", commitTreesTool)
}
return a.Root, a.Trees, nil
}
}
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
const getFileTool = "gitea.gitea_get_file"
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
// recognises it as /: 404 /).
var notFound = regexp.MustCompile(`: 404 `)
// filesVia asks the forge for a file through call. A file the forge says is not there is bridge.ErrNotFound.
func filesVia(ctx context.Context, call caller) bridge.Files {
return bridge.FilesVia(ctx, bridge.Caller(call))
return func(owner, repo, path, ref string) ([]byte, error) {
raw, err := call(ctx, getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
if err != nil {
if notFound.MatchString(err.Error()) {
return nil, fmt.Errorf("%w: %v", bridge.ErrNotFound, err)
}
return nil, err
}
var a struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
}
return []byte(*a.File.Content), nil
}
}
@@ -9,7 +9,6 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -121,8 +120,8 @@ func TestARefusedActIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1341/AC
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.refused" {
t.Fatalf("published on %v", pub.subjects)
}
m, err := forgemsg.DecodeForge(pub.data[0])
if err != nil || m.Kind != forgemsg.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" {
m, err := delivery.DecodeForge(pub.data[0])
if err != nil || m.Kind != delivery.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" {
t.Errorf("message %+v err %v", m, err)
}
f.refuse = nil
@@ -189,8 +188,8 @@ func TestARefusedMergeIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1342/
if len(f.calls) != 1 || len(pub.data) != 1 {
t.Fatalf("calls %d messages %d, want one each", len(f.calls), len(pub.data))
}
m, err := forgemsg.DecodeForge(pub.data[0])
if err != nil || m.Kind != forgemsg.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA ||
m, err := delivery.DecodeForge(pub.data[0])
if err != nil || m.Kind != delivery.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA ||
m.Act != "merge" || m.Words != "novox/mesh-delivery#12 is not merged: its head moved" {
t.Errorf("message %+v err %v", m, err)
}
@@ -1,61 +0,0 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-forge-bridge/internal/groups"
)
// endAsked answers one value of the state end-asks (the tool bridge_group_end_again, ADR 0633 §4): a value not yet answered
// says the group's end again from the gate's merges, with the ask's time appended to the message id so the stream's duplicate
// window does not drop it, and is written back by compare-and-set at its revision with whether it was written. A value
// answered already (this bridge's own write) is left.
func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte) {
a, err := groups.DecodeEndAsk(value)
if err != nil {
g.logger.Printf("%s.%s holds what is not an end ask, left: %v", groups.EndAsksState, key, err)
return
}
if a.Done() {
return
}
a = g.answerEnd(ctx, key, a, time.Now())
g.logger.Printf("the end of the group %q said again by %s (%s): %s", a.Group, a.By, a.Why, a.Answer)
data, err := json.Marshal(a)
if err != nil {
return
}
if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
g.logger.Printf("%s.%s: the answer could not be written: %v", groups.EndAsksState, key, err)
}
}
// readMerges reads one key of the gate's merges state.
func (g *grouper) readMerges(ctx context.Context, key string) ([]byte, uint64, bool, error) {
if g.merges == nil {
return nil, 0, false, errors.New("the bridge does not read the gate's merges (its bucket was not there at the start)")
}
value, rev, found, err := g.merges(ctx, key)
return value, rev, found, err
}
// answerEnd says the group's end again for an ask and returns the ask with its answer.
func (g *grouper) answerEnd(ctx context.Context, key string, a groups.EndAsk, at time.Time) groups.EndAsk {
a.Answered = at.UTC()
switch merges, mrev, found, err := g.readMerges(ctx, key); {
case err != nil:
a.Answer = "the gate's merges " + key + " cannot be read: " + err.Error()
case !found:
a.Answer = "the gate holds no merge of the group " + a.Group
case g.end(ctx, key, mrev, merges, groups.AgainStamp(a.At)):
a.Written, a.Answer = true, "the end of the group is written again"
default:
a.Answer = "the end is not written: a member is not merged or its merged message is not written, or FORGE refused it (the bridge's journal says which)"
}
return a
}
@@ -1,178 +0,0 @@
package main
import (
"context"
"errors"
"log"
"slices"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
groupkeys "mesh-delivery-forge-bridge/internal/groups"
)
// A delivery group's merges reach the indexer together (novox/hq ADR 0633 §4): the bridge sets the group on each
// member's merged message, from the gate's merges state it reads (ADR 0437 §8), and once every approved member is
// merged and its merged message written on FORGE it says the group's end, forge.group.merged, once.
//
// What it keeps of it is in its state performed, beside the gate's keys it acted on: `published.<hash>` for each
// member's merged message written (the hash is the first 16 hex digits of the sha256 of `<repository>#<number>@<merge>`),
// and `groups.<merges key>` once the group's end is written, its value the revision of the gate's merges key it was
// worked out from. A group whose end is not written yet is open (bridge_groups reads them).
// merges reads one key of the gate's merges state.
type mergesRead func(ctx context.Context, key string) (value []byte, rev uint64, found bool, err error)
// grouper sets a member's group and says a group's end.
type grouper struct {
merges mergesRead
send func(ctx context.Context, subject string, data []byte, id string) error // writes on FORGE with the message id
done revisions
logger *log.Logger
}
// sendVia writes on FORGE through pub, with the message id.
func sendVia(pub publisher) func(context.Context, string, []byte, string) error {
return func(ctx context.Context, subject string, data []byte, id string) error {
_, err := pub.Publish(ctx, subject, data, jetstream.WithMsgID(id))
return err
}
}
func kvMerges(kv jetstream.KeyValue) mergesRead {
return func(ctx context.Context, key string) ([]byte, uint64, bool, error) {
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, false, nil
}
if err != nil {
return nil, 0, false, err
}
return e.Value(), e.Revision(), true, nil
}
}
// groupOf is the gate's merge of the group a merged message's branch names, when the message's pull request is one of
// its approved members and the group has more than one: a lone pull request is no group.
func (g *grouper) groupOf(ctx context.Context, f forgemsg.Forge) (delivery.Merge, uint64, bool) {
if g == nil || g.merges == nil || f.Branch == "" {
return delivery.Merge{}, 0, false
}
key, err := delivery.MergesKey(f.Branch)
if err != nil {
return delivery.Merge{}, 0, false
}
value, rev, found, err := g.merges(ctx, key)
if err != nil {
g.logger.Printf("%s#%d: the gate's merges.%s cannot be read, so its merged message names no group: %v", f.Repository, f.Number, key, err)
return delivery.Merge{}, 0, false
}
if !found {
return delivery.Merge{}, 0, false
}
m, err := delivery.DecodeMerge(value)
if err != nil {
g.logger.Printf("%s#%d: the gate's merges.%s is not a merge, so its merged message names no group: %v", f.Repository, f.Number, key, err)
return delivery.Merge{}, 0, false
}
if _, ok := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; !ok || len(m.Approved) < 2 {
return delivery.Merge{}, 0, false
}
return m, rev, true
}
// member sets the group and its approved members on a merged message's body (ADR 0633 §4); any other body, and a
// merge of no group, is left as it is.
func (g *grouper) member(ctx context.Context, data []byte) ([]byte, forgemsg.Forge, bool) {
f, err := forgemsg.DecodeForge(data)
if err != nil || f.Kind != forgemsg.ForgeMerged {
return data, f, false
}
m, _, ok := g.groupOf(ctx, f)
if !ok {
return data, f, false
}
f.Group, f.Members = m.Group, groupkeys.MembersOf(m)
out, err := forgemsg.EncodeForge(f)
if err != nil {
g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.Group, err)
return data, f, false
}
return out, f, true
}
// published keeps that a member's merged message is written, and says the group's end when it was the last.
func (g *grouper) published(ctx context.Context, f forgemsg.Forge) {
if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number, f.Merge), 1); err != nil {
g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err)
return
}
key, err := delivery.MergesKey(f.Branch)
if err != nil {
return
}
value, rev, found, err := g.merges(ctx, key)
if err != nil || !found {
return
}
g.ended(ctx, key, rev, value)
}
// ended says the group's end (forge.group.merged) once every approved member of the gate's merge is merged and its
// merged message written, after the last of them; never on a refusal (ADR 0441 §3: a held group merges on from it).
// It is written once per group: `groups.<key>` in performed says it was. again says it is written again by hand
// (bridge_group_end_again), with the time appended to the message id, so the duplicate window does not drop it.
func (g *grouper) ended(ctx context.Context, key string, rev uint64, value []byte) {
g.end(ctx, key, rev, value, "")
}
func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, again string) bool {
m, err := delivery.DecodeMerge(value)
if err != nil || len(m.Approved) < 2 {
return false
}
if again == "" {
if _, seen, err := g.done.Get(ctx, groupkeys.Key(key)); err == nil && seen {
return false
}
}
end := forgemsg.ForgeGroupEnd{Group: m.Group}
for _, member := range groupkeys.MembersOf(m) {
at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool {
return delivery.SameRepository(p.Repository, member.Repository) && p.Number == member.Number
})
if at < 0 || m.Pulls[at].Commit == "" {
return false // not every member is merged (a merge commit is written only for a merged pull): the group stays gathered
}
if _, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok {
return false // its merged message is not written yet: said when it is (published)
}
end.Members = append(end.Members, forgemsg.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit})
}
// The members in the order the gate merged them, so the last commit names the end.
slices.SortStableFunc(end.Members, func(a, b forgemsg.GroupMerge) int {
return slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == a.Merge }) -
slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == b.Merge })
})
data, err := forgemsg.EncodeForgeGroupEnd(end)
if err != nil {
g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err)
return false
}
id := groupkeys.EndID(m.Group, end.Members[len(end.Members)-1].Merge, "")
if again != "" {
id += "." + again
}
if err := g.send(ctx, forgeSubject(forgemsg.ForgeGroupEndEvent), data, id); err != nil {
g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err)
return false
}
if err := g.done.Put(ctx, groupkeys.Key(key), rev); err != nil {
g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.Group, err)
}
g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.Group, len(end.Members), end.Members[len(end.Members)-1].Merge)
return true
}
@@ -1,141 +0,0 @@
package main
import (
"bytes"
"context"
"log"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
groupkeys "mesh-delivery-forge-bridge/internal/groups"
)
// idPub records the subjects, bodies and message ids it is handed.
type idPub struct {
subjects, ids []string
data [][]byte
}
func (p *idPub) send(_ context.Context, subject string, data []byte, id string) error {
p.subjects, p.data, p.ids = append(p.subjects, subject), append(p.data, data), append(p.ids, id)
return nil
}
func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17, T19 (ADR 0633 §4)
ctx := context.Background()
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
merge := func(states ...delivery.MergePull) []byte {
v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Pulls: states,
Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}})
if err != nil {
t.Fatal(err)
}
return v
}
sdk := func(state delivery.MergeState, commit, words string) delivery.MergePull {
return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-sdk", Number: 37, Head: h1}, State: state, Commit: commit, Words: words}
}
dlv := func(state delivery.MergeState, commit string) delivery.MergePull {
return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: state, Commit: commit}
}
gate := map[string][]byte{}
var logs bytes.Buffer
pub, done := &idPub{}, memRevisions{}
g := &grouper{send: pub.send, done: done, logger: log.New(&logs, "", 0),
merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) {
v, ok := gate[key]
return v, 7, ok, nil
}}
mergedOf := func(repo string, number int, head, commit string) []byte {
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"})
if err != nil {
t.Fatal(err)
}
return data
}
// A lone pull request's merged message carries no group.
if _, f, ok := g.member(ctx, mergedOf("novox/hq", 4, h1, c1)); ok || f.Group != "" {
t.Errorf("a lone pull request names the group %q", f.Group)
}
// The first member: its message names the group and both members; the group is not ended.
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerging, ""))
data, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1))
if back, err := forgemsg.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 {
t.Fatalf("a member's message: %+v %v", back, err)
}
g.published(ctx, f)
// A refusal of the other member is no end (ADR 0441 §3: the group merges on from it later).
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: delivery.MergeRefused, Words: "405"})
g.ended(ctx, "feat/take-in", 8, gate["feat/take-in"])
// Merged on at the gate before its merged message is written: still no end; then its message ends the group.
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerged, c2))
g.ended(ctx, "feat/take-in", 9, gate["feat/take-in"])
if len(pub.subjects) != 0 {
t.Fatalf("an end before every member's merged message is written: %v", pub.subjects)
}
_, f, _ = g.member(ctx, mergedOf("novox/mesh-delivery", 9, h2, c2))
g.published(ctx, f)
g.ended(ctx, "feat/take-in", 10, gate["feat/take-in"]) // a later write of the gate says no second end
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.group.merged" || pub.ids[0] != "group-end.feat/take-in."+c2 {
t.Fatalf("ends %v ids %v, want one forge.group.merged with id group-end.feat/take-in.%s", pub.subjects, pub.ids, c2)
}
end, err := forgemsg.DecodeForgeGroupEnd(pub.data[0])
if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 {
t.Errorf("the end = %+v %v, want both members in their merge order", end, err)
}
if rev, ok := done[groupkeys.Key("feat/take-in")]; !ok || rev != 7 {
t.Errorf("performed %v: want groups.feat/take-in kept", done)
}
// Again by hand: written once more, its id not the first's.
if !g.end(ctx, "feat/take-in", 10, gate["feat/take-in"], "20261104T2300") || pub.ids[1] == pub.ids[0] {
t.Errorf("the end again by hand: ids %v", pub.ids)
}
// 1585/T19: an ask to say a group's end again is answered by the bridge from the gate's merges, with the ask's time on
// the message id; not written while a member is not merged, with the reason; not for a group the gate does not hold.
pull := func(repo string, n int, head string, state delivery.MergeState, commit string) delivery.MergePull {
return delivery.MergePull{Head: delivery.Head{Repository: repo, Number: n, Head: head}, State: state, Commit: commit}
}
askMerge := func(second delivery.MergeState, commit2 string) []byte {
v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2},
Pulls: []delivery.MergePull{pull("novox/mesh-sdk", 37, h1, delivery.MergeMerged, c1), pull("novox/mesh-delivery", 9, h2, second, commit2)}})
if err != nil {
t.Fatal(err)
}
return v
}
ask := groupkeys.EndAsk{V: 1, Group: "feat/take-in", By: "operator@shanks", Why: "the indexer never took it in", At: time.Date(2026, 11, 4, 23, 0, 0, 123, time.UTC)}
for _, tc := range []struct {
name string
gate map[string][]byte
published bool
written bool
answer string
}{
{"merged and published: written again", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, true, true, "written again"},
{"a member's message not written", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, false, false, "not written"},
{"a member not merged", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerging, "")}, true, false, "not written"},
{"no merge at the gate", map[string][]byte{}, true, false, "holds no merge"},
} {
pub, done := &idPub{}, memRevisions{}
if tc.published {
done[groupkeys.PublishedKey("novox/mesh-sdk", 37, c1)], done[groupkeys.PublishedKey("novox/mesh-delivery", 9, c2)] = 1, 1
}
g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0),
merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) {
v, ok := tc.gate[key]
return v, 4, ok, nil
}}
got := g.answerEnd(ctx, "feat/take-in", ask, ask.At.Add(time.Second))
if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] {
t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids)
}
if tc.written && pub.ids[0] != groupkeys.EndID("feat/take-in", c2, groupkeys.AgainStamp(ask.At)) {
t.Errorf("%s: the id %s does not carry the ask's time", tc.name, pub.ids[0])
}
}
}
@@ -20,7 +20,6 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -37,7 +36,7 @@ import (
//
// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls,
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, performed, inputs and end-asks (ADR 0633 §5's hand end), and the stream forge; `**` is written `>` on the wire (modstreams.go
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
// WirePattern). The consumer's durable is <node>_<module> on EVENTS (nats.go consumerDurable). The controller's own
// user keeps all rights, as it has.
const busConf = `
@@ -84,12 +83,6 @@ accounts {
"$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>"
"$KV.mesh-delivery-forge-bridge_performed.>"
"$KV.mesh-delivery-forge-bridge_inputs.>"
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_end-asks.>"
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_end-asks.>"
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_end-asks.>"
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_end-asks"
"$JS.FC.KV_mesh-delivery-forge-bridge_end-asks.>"
"$KV.mesh-delivery-forge-bridge_end-asks.>"
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>"
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>"
"$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>"
@@ -328,13 +321,13 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2
if err != nil {
t.Fatal(err)
}
f, err := forgemsg.DecodeForge(raw.Data)
f, err := delivery.DecodeForge(raw.Data)
if err != nil {
t.Fatal(err)
}
want := forgemsg.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
if !forgemsg.SameForge(f, want) {
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
@@ -423,16 +416,16 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
if err != nil {
t.Fatal(err)
}
f, err := forgemsg.DecodeForge(raw.Data)
f, err := delivery.DecodeForge(raw.Data)
if err != nil {
t.Fatal(err)
}
want := forgemsg.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
// No mesh package: only the indexer computes what a fingerprint takes (ADR 0593 §2).
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
if !forgemsg.SameForge(f, want) {
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
select {
@@ -485,7 +478,7 @@ func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1
if err != nil {
t.Fatal(err)
}
if f, err := forgemsg.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
if f, err := delivery.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
t.Errorf("%s: %+v err %v", repo, f, err)
}
}
@@ -555,20 +548,20 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes
t.Fatalf("FORGE holds %d messages (%v), want %d", info.State.Msgs, err, want)
}
}
named := func(seq uint64) []forgemsg.OpenPull {
named := func(seq uint64) []delivery.OpenPull {
t.Helper()
raw, err := forge.GetMsg(b.ctx, seq)
if err != nil || raw.Subject != openSubject {
t.Fatalf("message %d: %v %v, want the open message", seq, raw, err)
}
f, err := forgemsg.DecodeForge(raw.Data)
f, err := delivery.DecodeForge(raw.Data)
if err != nil {
t.Fatal(err)
}
return f.Open
}
bothOpen := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
same := func(got []forgemsg.OpenPull) bool {
bothOpen := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
same := func(got []delivery.OpenPull) bool {
return len(got) == 2 && got[0] == bothOpen[0] && got[1] == bothOpen[1]
}
@@ -720,7 +713,7 @@ func forgeMerged(t *testing.T, b *bed) []string {
if err != nil {
continue
}
if f, err := forgemsg.DecodeForge(raw.Data); err == nil && f.Kind == forgemsg.ForgeMerged {
if f, err := delivery.DecodeForge(raw.Data); err == nil && f.Kind == delivery.ForgeMerged {
out = append(out, f.Repository)
}
}
@@ -21,11 +21,9 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-forge-bridge/internal/bridge"
groupkeys "mesh-delivery-forge-bridge/internal/groups"
)
// eventsStream is the shared stream every module's events are on; the module's consumer on it is the durable
@@ -70,18 +68,16 @@ type publisher interface {
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
// delivers it again (the server's ack wait is the only clock).
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, logger *log.Logger) {
handleGrouped(ctx, m, pub, trees, files, kept, nil, logger)
}
// handleGrouped is handle; a merged message of a delivery group's member names its group (g, ADR 0633 §4; nil names
// none), and the member's merged message written may end the group.
func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, g *grouper, logger *log.Logger) {
msg, err := bridge.ConvertKept(m.Subject(), m.Data(), trees, files, kept)
switch {
case errors.Is(err, bridge.ErrRetry):
logger.Printf("%v; heard again in %s", err, retryDelay)
_ = m.NakWithDelay(retryDelay)
return
case errors.Is(err, bridge.ErrNoModules):
logger.Printf("%v; no merged message", err)
_ = m.Ack()
return
case errors.Is(err, bridge.ErrNotHandled):
logger.Printf("%v; nothing written", err)
_ = m.Ack()
@@ -96,18 +92,10 @@ func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge
if id := m.Headers().Get("x-event-id"); id != "" {
opts = append(opts, jetstream.WithMsgID(id))
}
var member forgemsg.Forge
grouped := false
if g != nil {
msg.Data, member, grouped = g.member(ctx, msg.Data)
}
if _, err := pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, opts...); err != nil {
logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err)
return
}
if grouped {
g.published(ctx, member)
}
if msg.Said != "" {
logger.Printf("%s: %s", m.Subject(), msg.Said)
}
@@ -152,25 +140,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
}
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
// A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the gate's merges.
groups := &grouper{send: sendVia(js), done: kvRevisions{performedKV}, logger: logger}
watching := make(chan error, 3)
for _, w := range []struct {
bucket string
react func(ctx context.Context, key string, rev uint64, value []byte)
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, func(ctx context.Context, key string, rev uint64, value []byte) {
perf.merges(ctx, key, rev, value)
groups.ended(ctx, key, rev, value)
}}} {
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945).
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf)
if kv == nil {
continue
}
if w.bucket == delivery.MergesBucket {
groups.merges = kvMerges(kv)
}
go func() {
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
@@ -179,16 +159,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
})
}()
}
// A person says a group's end again with bridge_group_end_again, which writes the state end-asks (ADR 0633 §4): watched, and
// answered there; a state not there at this start is answered from the next start.
if kv := process.OptionalBucket(ctx, js, groupkeys.EndAsksBucket, "bridge_group_end_again is not answered until the bridge starts again", logger.Printf); kv != nil {
go func() {
err := watch(ctx, kv, func(key string, rev uint64, value []byte) { groups.endAsked(ctx, kv, key, rev, value) })
if err != nil && ctx.Err() == nil {
logger.Printf("the end asks are no longer watched, so bridge_group_end_again is not answered until the bridge starts again: %v", err)
}
}()
}
if err := said(ctx); err != nil {
return err
}
@@ -205,7 +175,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Bounded: the turn is held, and no sync or other event goes on without it.
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
defer cancel()
handleGrouped(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, groups, logger)
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, logger)
})
}, logger.Printf)
if err != nil {
@@ -18,7 +18,7 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-forge-bridge/internal/bridge"
)
@@ -85,7 +85,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
t.Fatalf("published on %v", pub.subjects)
}
f, err := forgemsg.DecodeForge(pub.data[0])
f, err := delivery.DecodeForge(pub.data[0])
if err != nil || f.Number != 12 || f.Branch != "feat/x" || f.Head != head || f.Action != "updated" || !f.DirsKnown {
t.Errorf("message %+v err %v", f, err)
}
@@ -107,10 +107,11 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
if !strings.Contains(logs.String(), "mesh.mod.gitea.event.pull.updated") || !strings.Contains(logs.String(), "head") {
t.Errorf("log %q names neither the subject nor why", logs.String())
}
// 1235: an unwritten message is not acked (a failed write is redelivered by the server, not retried here); merged here
// from a test of its own, for the test budget.
logs.Reset()
m, pub = &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
}
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
if m.acked != 0 || m.termd != 0 {
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
@@ -172,7 +173,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged" {
t.Fatalf("published on %v", pub.subjects)
}
f, err := forgemsg.DecodeForge(pub.data[0])
f, err := delivery.DecodeForge(pub.data[0])
if err != nil || f.Root != "r0" || f.Trees[""] != "t0" || f.Trees["modules/a"] != "t1" || f.Merge == "" {
t.Errorf("message %+v err %v", f, err)
}
@@ -10,7 +10,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-forge-bridge/internal/bridge"
)
@@ -41,7 +41,7 @@ func (r *recorder) Publish(ctx context.Context, subject string, payload []byte,
if err != nil {
return ack, err
}
f, derr := forgemsg.DecodeForge(payload)
f, derr := delivery.DecodeForge(payload)
if derr != nil {
return ack, nil
}
@@ -101,7 +101,7 @@ func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore,
}
var differing []bridge.Message
for _, m := range msgs {
f, err := forgemsg.DecodeForge(m.Data)
f, err := delivery.DecodeForge(m.Data)
if err != nil {
differing = append(differing, m)
continue
@@ -13,7 +13,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds its entries inside {pulls}.
@@ -43,7 +43,7 @@ func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1,
}
want := []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.pull"}
for i, w := range want {
f, err := forgemsg.DecodeForge(pub.data[i])
f, err := delivery.DecodeForge(pub.data[i])
if pub.subjects[i] != w || err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
t.Errorf("message %d on %s = %+v err %v", i, pub.subjects[i], f, err)
}
@@ -112,8 +112,8 @@ func TestSyncOpenWritesPullsForNewHeadsThenOneOpenMessageNamingAll(t *testing.T)
if pub.subjects[1] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.open" {
t.Fatalf("last message on %s, want the open message after the pull messages", pub.subjects[1])
}
f, err := forgemsg.DecodeForge(pub.data[1])
want := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
f, err := delivery.DecodeForge(pub.data[1])
want := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
if err != nil || len(f.Open) != 2 || f.Open[0] != want[0] || f.Open[1] != want[1] {
t.Errorf("open message = %+v err %v, want it to name %v, the pull request whose head did not differ among them", f, err, want)
}
@@ -140,10 +140,10 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6
pub := &fakePub{}
heads := newHeads()
r := &recorder{pub: pub, heads: heads, logger: log.New(&bytes.Buffer{}, "", 0)}
write := func(f forgemsg.Forge) {
write := func(f delivery.Forge) {
t.Helper()
ev, _ := forgemsg.ForgeEvent(f.Repository, f.Kind)
raw, err := forgemsg.EncodeForge(f)
ev, _ := delivery.ForgeEvent(f.Repository, f.Kind)
raw, err := delivery.EncodeForge(f)
if err != nil {
t.Fatal(err)
}
@@ -151,19 +151,19 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6
t.Fatal(err)
}
}
p := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated}
p := delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated}
write(p)
if heads.get("novox.mesh-delivery.12") != head {
t.Fatalf("bucket %v", heads.vals)
}
p.Action = forgemsg.PullClosed
p.Action = delivery.PullClosed
write(p)
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
t.Errorf("a closed pull request is still kept: %v", heads.vals)
}
p.Action = forgemsg.PullUpdated
p.Action = delivery.PullUpdated
write(p)
write(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
write(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
t.Errorf("a merged pull request is still kept: %v", heads.vals)
}
@@ -1,413 +0,0 @@
// Package forgemsg is the forge bridge's contract on its own stream `forge` (novox/hq ADR 0633 §4): its messages (Forge,
// one per pull request, merge, refusal or list of open pull requests) and a delivery group's end (ForgeGroupEnd). The
// forge bridge writes them; the indexer and the gate read them. It is the forge bridge's own package: one module owns
// these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04).
package forgemsg
import (
"encoding/json"
"fmt"
"reflect"
"regexp"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Version is the version of every value this package writes and the only one it reads: mesh-sdk's delivery values'.
const Version = delivery.Version
// ForgeStream is the forge bridge's stream as its manifest declares it; the broker derives the real names.
const ForgeStream = "forge"
// The kinds of forge message, the last part of a forge event's name.
const (
ForgePull = "pull"
ForgeMerged = "merged"
ForgeRefused = "refused"
ForgeOpen = "open"
)
// OpenPull names an open pull request into the trunk: the repository as `<owner>/<name>` and the number.
type OpenPull struct {
Repository string `json:"repository"`
Number int `json:"number"`
}
// OpenEvent is the name of an open message: `forge.open`. It names no repository: it says every open pull request of
// every repository the forge bridge reads (novox/hq ADR 0443 §5).
func OpenEvent() string { return "forge." + ForgeOpen }
// The actions of a pull message.
const (
PullOpened = "opened"
PullUpdated = "updated"
PullClosed = "closed"
)
// checkRepository refuses a repository that is not `<owner>/<name>`: exactly one slash, each part a valid name.
func checkRepository(repository string) error {
owner, name, ok := strings.Cut(repository, "/")
if !ok || strings.Contains(name, "/") {
return fmt.Errorf("delivery: the repository %q is not <owner>/<name>, with exactly one slash", repository)
}
if err := checkName("repository owner", owner); err != nil {
return fmt.Errorf("delivery: the repository %q: %w", repository, err)
}
if err := checkName("repository name", name); err != nil {
return fmt.Errorf("delivery: the repository %q: %w", repository, err)
}
if strings.Contains(owner, "/") {
return fmt.Errorf("delivery: the repository %q is not <owner>/<name>, with exactly one slash", repository)
}
return nil
}
func checkForgeKind(kind string) error {
switch kind {
case ForgePull, ForgeMerged, ForgeRefused, ForgeOpen:
return nil
}
return fmt.Errorf("delivery: the forge kind %q is not pull, merged, refused or open", kind)
}
// ForgeEvent is the name of a forge message of the given kind: `forge.<owner>.<name>.<kind>`.
func ForgeEvent(repository, kind string) (string, error) {
if err := checkForgeKind(kind); err != nil {
return "", err
}
if kind == ForgeOpen {
return "", fmt.Errorf("delivery: an open message names no repository, its event is %s", OpenEvent())
}
if err := checkRepository(repository); err != nil {
return "", err
}
owner, name, _ := strings.Cut(repository, "/")
return "forge." + owner + "." + name + "." + kind, nil
}
// ParseForgeEvent is the inverse of ForgeEvent.
func ParseForgeEvent(event string) (repository, kind string, err error) {
if event == OpenEvent() {
return "", ForgeOpen, nil
}
parts := strings.Split(event, ".")
if len(parts) != 4 || parts[0] != "forge" {
return "", "", fmt.Errorf("delivery: %q is not a forge event, forge.<owner>.<name>.<kind>", event)
}
repository = parts[1] + "/" + parts[2]
if _, err := ForgeEvent(repository, parts[3]); err != nil {
return "", "", fmt.Errorf("delivery: %q is not a forge event: %w", event, err)
}
return repository, parts[3], nil
}
// Forge is a message of the forge bridge's stream, trimmed to repository, branch, pull request and head. A pull
// message says Action; a merged message carries the merge commit, the tree of every directory holding a
// module.json and the repository's root tree; a refused message names the owed act it answers and the forge's own
// words. A pull message may also carry the module directories its head changes (Dirs, in the order the forge said) and
// DirsKnown, true when the forge could say them: known and empty is DirsKnown with no Dirs, not known is neither
// (the forge's module_dirs and module_dirs_said, mesh-catalog modules/gitea/trunk.ts). Other messages carry neither.
type Forge struct {
V int `json:"v"`
Kind string `json:"kind"`
Repository string `json:"repository"`
Branch string `json:"branch,omitempty"`
Number int `json:"number,omitempty"`
Head string `json:"head,omitempty"`
Action string `json:"action,omitempty"`
Dirs []string `json:"dirs,omitempty"`
DirsKnown bool `json:"dirs-known,omitempty"`
Merge string `json:"merge,omitempty"`
Trees map[string]string `json:"trees,omitempty"`
Root string `json:"root,omitempty"`
// Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message,
// of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path
// relative to it (novox/hq ADR 0465 clause 1). Older writers send neither.
Manifests map[string]string `json:"manifests,omitempty"`
Locks map[string]map[string]string `json:"locks,omitempty"`
// Paths are, of each module directory whose manifest names paths on every artifact, each named path's git object
// id at the same commit as its manifest, by path relative to the directory (novox/hq ADR 0499 clause 3). The
// indexer names the paths; a message without them gives such a module none. Older writers send none.
Paths map[string]map[string]string `json:"paths,omitempty"`
// Packages are, of each module directory, the mesh packages its lock file names (MeshPackagesOf; novox/hq ADR 0573
// §2): the indexer keeps them in the module's inputs and takes each one's latest published version into its
// fingerprint. Absent for a directory that uses none, and from an older writer.
Packages map[string][]string `json:"packages,omitempty"`
Act string `json:"act,omitempty"`
Words string `json:"words,omitempty"`
Open []OpenPull `json:"open,omitempty"`
// RetiredPlacements is the placement files a bridge before ADR 0567 carried (ADR 0549 §2, ADR 0559 §2): placement is
// the seat mesh-placement's state now and no repository holds it (ADR 0567 §11), so a message carrying them is
// refused, loudly, rather than read with them dropped.
RetiredPlacements json.RawMessage `json:"placements,omitempty"`
RetiredRead bool `json:"placements-read,omitempty"`
// Predefined is the catalogue's predefined.json, exact, at the merge commit of a merged message (novox/hq ADR 0565
// §2): what the indexer keeps as its state `predefined`. PredefinedRead says the bridge read the merge and it holds
// no predefined.json. Carried on a merged message only; an older bridge sends neither, and the indexer keeps what it
// held.
Predefined json.RawMessage `json:"predefined,omitempty"`
PredefinedRead bool `json:"predefined-read,omitempty"`
// Group is the delivery group a merged message's pull request was merged in, and Members every approved member of
// that group with its approved head, this one included (novox/hq ADR 0633 §4): the forge bridge sets both from the
// gate's merges state, and the indexer gathers the group's merges until the group's end (ForgeGroupEnd). A merge
// with no group carries neither, and an older bridge sends neither; an older reader ignores both.
Group string `json:"group,omitempty"`
Members []delivery.Head `json:"members,omitempty"`
}
// checkGroup refuses a group on anything but a merged message, a group without its members or members without a
// group, a member that is not a repository's pull request at a head, a member named twice, and members that leave out
// the message's own pull request.
func (f Forge) checkGroup() error {
if f.Group == "" && len(f.Members) == 0 {
return nil
}
if f.Kind != ForgeMerged {
return fmt.Errorf("delivery: only a merged message names its delivery group, not a %s message", f.Kind)
}
if f.Group == "" || len(f.Members) == 0 {
return fmt.Errorf("delivery: a merged message names its group and the group's members together, or neither")
}
self := false
seen := map[string]bool{}
for _, m := range f.Members {
if err := checkRepository(m.Repository); err != nil {
return err
}
if m.Number <= 0 || m.Head == "" {
return fmt.Errorf("delivery: the member %s of the group %q needs its pull request number and head", m.Repository, f.Group)
}
k := fmt.Sprintf("%s#%d", m.Repository, m.Number)
if seen[k] {
return fmt.Errorf("delivery: the group %q names its member %s twice", f.Group, k)
}
seen[k] = true
self = self || (delivery.SameRepository(m.Repository, f.Repository) && m.Number == f.Number)
}
if !self {
return fmt.Errorf("delivery: the members of the group %q leave out the merged pull request %s#%d", f.Group, f.Repository, f.Number)
}
return nil
}
// checkPredefined refuses the predefined list on anything but a merged message, a list that is not one JSON object, and
// a list said both present and absent.
func (f Forge) checkPredefined() error {
if len(f.Predefined) == 0 && !f.PredefinedRead {
return nil
}
if f.Kind != ForgeMerged {
return fmt.Errorf("delivery: only a merged message carries the predefined list, not a %s message", f.Kind)
}
if len(f.Predefined) > 0 && f.PredefinedRead {
return fmt.Errorf("delivery: a merged message carries the predefined list and says the merge holds none (predefined-read)")
}
var list map[string]json.RawMessage
if len(f.Predefined) > 0 && json.Unmarshal(f.Predefined, &list) != nil {
return fmt.Errorf("delivery: the predefined list a merged message carries is not one JSON object")
}
return nil
}
func (f Forge) check() error {
if err := checkForgeKind(f.Kind); err != nil {
return err
}
if f.Kind == ForgeOpen {
return f.checkOpen()
}
if len(f.Open) > 0 {
return fmt.Errorf("delivery: only an open message names open pull requests")
}
if len(f.RetiredPlacements) > 0 || f.RetiredRead {
return fmt.Errorf("delivery: a forge message carries placement files, which no repository holds since ADR 0567: " +
"placement is the seat mesh-placement's state")
}
if err := f.checkPredefined(); err != nil {
return err
}
if err := f.checkGroup(); err != nil {
return err
}
if err := checkRepository(f.Repository); err != nil {
return err
}
switch f.Kind {
case ForgePull:
if f.Number <= 0 {
return fmt.Errorf("delivery: a pull message needs its pull request number")
}
if f.Head == "" {
return fmt.Errorf("delivery: a pull message needs its head")
}
switch f.Action {
case PullOpened, PullUpdated, PullClosed:
default:
return fmt.Errorf("delivery: the pull action %q is not opened, updated or closed", f.Action)
}
if len(f.Dirs) > 0 && !f.DirsKnown {
return fmt.Errorf("delivery: a pull message with module directories must say it knows them (dirs-known)")
}
dirs := map[string]bool{}
for _, d := range f.Dirs {
dirs[d] = true
}
if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil {
return err
}
case ForgeMerged:
if len(f.Dirs) > 0 || f.DirsKnown {
return fmt.Errorf("delivery: a merged message carries no module directories")
}
if f.Number <= 0 {
return fmt.Errorf("delivery: a merged message needs its pull request number")
}
if f.Head == "" {
return fmt.Errorf("delivery: a merged message needs its head")
}
if f.Merge == "" {
return fmt.Errorf("delivery: a merged message needs its merge commit")
}
if f.Root == "" {
return fmt.Errorf("delivery: a merged message needs the repository's root tree")
}
// No trees is a merge that leaves the repository with no module.json anywhere (a documents repository, or one
// whose last module was removed): it is written all the same, so the indexer takes the merge in and drops the
// repository's modules (novox/hq issue 1126). An older writer always sent at least one tree, so none is unambiguous;
// such a message carries no manifests, locks or paths (checkInputs below: each names a directory of its trees).
for dir, tree := range f.Trees {
if tree == "" {
return fmt.Errorf("delivery: a merged message's tree of the directory %q is empty", dir)
}
}
if err := f.checkInputs(func(d string) bool { _, ok := f.Trees[d]; return ok }, "its trees"); err != nil {
return err
}
case ForgeRefused:
if len(f.Dirs) > 0 || f.DirsKnown {
return fmt.Errorf("delivery: a refused message carries no module directories")
}
if len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 {
return fmt.Errorf("delivery: a refused message carries no manifests, lock hashes or named paths")
}
if f.Act == "" {
return fmt.Errorf("delivery: a refused message needs the act it answers")
}
if f.Words == "" {
return fmt.Errorf("delivery: a refused message needs the forge's words")
}
}
return nil
}
// checkInputs refuses a manifest or a lock hash of a directory that known does not know (of is what it knows them
// by), an empty manifest and an empty lock path or hash, naming the directory.
func (f Forge) checkInputs(known func(dir string) bool, of string) error {
for dir, m := range f.Manifests {
if !known(dir) {
return fmt.Errorf("delivery: the manifest of the directory %q is not of %s", dir, of)
}
if m == "" {
return fmt.Errorf("delivery: the manifest of the directory %q is empty", dir)
}
}
for dir, locks := range f.Locks {
if !known(dir) {
return fmt.Errorf("delivery: the lock hashes of the directory %q are not of %s", dir, of)
}
for path, hash := range locks {
if path == "" || hash == "" {
return fmt.Errorf("delivery: the lock %q of the directory %q has an empty path or hash", path, dir)
}
}
}
for dir, paths := range f.Paths {
if !known(dir) {
return fmt.Errorf("delivery: the named paths of the directory %q are not of %s", dir, of)
}
for path, object := range paths {
if path == "" || object == "" {
return fmt.Errorf("delivery: the named path %q of the directory %q has an empty path or object", path, dir)
}
}
}
return nil
}
// checkOpen refuses an open message that names a repository, a branch, a head or anything of the other kinds, or
// an open pull request that is not `<owner>/<name>` and a number.
func (f Forge) checkOpen() error {
if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown ||
f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 || f.Root != "" || f.Act != "" || f.Words != "" || f.Group != "" || len(f.Members) > 0 {
return fmt.Errorf("delivery: an open message carries only the open pull requests it names")
}
for _, p := range f.Open {
if err := checkRepository(p.Repository); err != nil {
return err
}
if p.Number <= 0 {
return fmt.Errorf("delivery: the open pull request of %s has no number", p.Repository)
}
}
return nil
}
// EncodeForge is f as JSON at Version; it refuses a message missing what its kind needs.
func EncodeForge(f Forge) ([]byte, error) {
if err := f.check(); err != nil {
return nil, err
}
f.V = Version
return json.Marshal(f)
}
// DecodeForge reads a forge message; it refuses an unknown version (ErrUnknownVersion) and what EncodeForge refuses.
func DecodeForge(data []byte) (Forge, error) {
var f Forge
if err := decode(data, &f); err != nil {
return Forge{}, err
}
if err := f.check(); err != nil {
return Forge{}, err
}
return f, nil
}
// SameForge says whether two forge messages are the same value.
func SameForge(a, b Forge) bool { return reflect.DeepEqual(a, b) }
var nameOK = regexp.MustCompile(`^[-/_=a-zA-Z0-9]+$`)
// checkName refuses a name that is empty or holds a dot, a wildcard, whitespace or a character a bus key refuses
// (mesh-sdk delivery's rule for a name).
func checkName(kind, name string) error {
if name == "" {
return fmt.Errorf("delivery: the %s name is empty", kind)
}
if !nameOK.MatchString(name) {
return fmt.Errorf("delivery: the %s name %q holds a dot, a wildcard, whitespace or a character a key refuses", kind, name)
}
return nil
}
// decode reads a value of Version, refusing another or none with delivery.ErrUnknownVersion, as mesh-sdk delivery's
// values are read.
func decode(data []byte, into any) error {
var probe struct {
V *int `json:"v"`
}
if err := json.Unmarshal(data, &probe); err != nil {
return fmt.Errorf("delivery: value is not JSON: %w", err)
}
if probe.V == nil {
return fmt.Errorf("%w: no version in the value, this reader reads %d", delivery.ErrUnknownVersion, Version)
}
if *probe.V != Version {
return fmt.Errorf("%w: the value is version %d, this reader reads %d", delivery.ErrUnknownVersion, *probe.V, Version)
}
if err := json.Unmarshal(data, into); err != nil {
return fmt.Errorf("delivery: value does not fit its shape: %w", err)
}
return nil
}
@@ -1,193 +0,0 @@
package forgemsg
import (
"encoding/json"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const oddManifest = "{\n\t\"module\": \"a\",\n \"z\": 1, \"a\": [ \"x\",\"y\" ],\n \"why\": \"é — <b> & \\n\"\n}\n\n"
func sampleForges() []Forge {
return []Forge{
{Kind: ForgePull, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 36, Head: "aaaa", Action: "opened"},
{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc",
Trees: map[string]string{"modules/a": "t1"}, Root: "t0"},
{Kind: ForgeRefused, Repository: "novox/mesh-sdk", Act: "novox/mesh-sdk.36.aaaa.view", Words: "403 forbidden"},
{Kind: ForgeOpen, Open: []OpenPull{{Repository: "novox/mesh-sdk", Number: 36}, {Repository: "novox/mesh-control", Number: 7}}},
// 1585/T1: a group member's merged message names its group and every approved member (ADR 0633 §4).
{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 37, Head: "dddd", Merge: "eeee", Root: "t0",
Group: "feat/take-in", Members: []delivery.Head{{Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}, {Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}}},
}
}
func mergedWithInputs() Forge {
return Forge{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc", Root: "t0",
Trees: map[string]string{"a": "t1", "b": "t2"},
Manifests: map[string]string{"a": oddManifest},
Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("ab", 32)}}}
}
// TestForge holds the forge bridge's contract, moved whole from mesh-sdk go/delivery with its tests (1200/AC1, AC3,
// AC4; 1279/AC5; 1442/AC1-AC4; 1585/T1).
func TestForge(t *testing.T) {
t.Run("events", func(t *testing.T) {
ev, err := ForgeEvent("novox/mesh-sdk", ForgeMerged)
if err != nil || ev != "forge.novox.mesh-sdk.merged" {
t.Fatalf("forge event = %q, %v", ev, err)
}
if repo, kind, err := ParseForgeEvent(ev); err != nil || repo != "novox/mesh-sdk" || kind != ForgeMerged {
t.Fatalf("parse forge event = %q %q %v", repo, kind, err)
}
for _, bad := range []struct{ repo, kind string }{
{"", "pull"}, {"novox", "pull"}, {"a/b/c", "pull"}, {"/b", "pull"}, {"a/", "pull"},
{"a.b/c", "pull"}, {"a/b c", "pull"}, {"a/b", "closed"}, {"a/b", ""}, {"a/b", ForgeOpen},
} {
if _, err := ForgeEvent(bad.repo, bad.kind); err == nil {
t.Errorf("forge event %q %q accepted", bad.repo, bad.kind)
}
}
for _, bad := range []string{"", "forge.a.b", "forge.a.b.closed", "x.a.b.pull", "forge.a.b.pull.x", "forge..b.pull", ForgeGroupEndEvent} {
if _, _, err := ParseForgeEvent(bad); err == nil {
t.Errorf("parse forge event %q accepted", bad)
}
}
if repo, kind, err := ParseForgeEvent(OpenEvent()); OpenEvent() != "forge.open" || err != nil || repo != "" || kind != ForgeOpen {
t.Errorf("open event %q = %q %q %v", OpenEvent(), repo, kind, err)
}
})
t.Run("round trip", func(t *testing.T) {
fs := append(sampleForges(), mergedWithInputs())
for _, f := range fs {
data, err := EncodeForge(f)
if err != nil {
t.Fatalf("%s: %v", f.Kind, err)
}
var probe map[string]any
if err := json.Unmarshal(data, &probe); err != nil || probe["v"] != float64(1) {
t.Fatalf("%s: json %s does not carry v:1", f.Kind, data)
}
f.V = Version
if got, err := DecodeForge(data); err != nil || !SameForge(got, f) {
t.Errorf("forge %s round trip: %+v %v", f.Kind, got, err)
}
}
// An empty list is an open message too: nothing is open.
if data, err := EncodeForge(Forge{Kind: ForgeOpen, Open: []OpenPull{}}); err != nil {
t.Errorf("empty open: %v", err)
} else if f, err := DecodeForge(data); err != nil || f.Kind != ForgeOpen || len(f.Open) != 0 {
t.Errorf("empty open round trip: %+v %v", f, err)
}
ge := ForgeGroupEnd{Group: "feat/take-in", Members: []GroupMerge{{Repository: "novox/mesh-sdk", Number: 37, Merge: "eeee"}}}
data, err := EncodeForgeGroupEnd(ge)
ge.V = Version
if got, err2 := DecodeForgeGroupEnd(data); err != nil || err2 != nil || !SameForgeGroupEnd(got, ge) {
t.Errorf("group end round trip: %+v %v %v", got, err, err2)
}
// An older bridge's message reads, and is written back byte for byte.
old := `{"v":1,"kind":"merged","repository":"novox/mesh-sdk","number":36,"head":"aaaa","merge":"cccc","trees":{"a":"t1"},"root":"t0"}`
if f, err := DecodeForge([]byte(old)); err != nil || f.Manifests != nil {
t.Fatalf("%+v, %v", f, err)
} else if data, err := EncodeForge(f); err != nil || string(data) != old {
t.Errorf("%s, %v, want exactly %s", data, err, old)
}
// A merge leaving the repository with no module.json is a message too (issue 1126).
f := sampleForges()[1]
f.Trees = nil
if data, err := EncodeForge(f); err != nil {
t.Errorf("a merged message with no trees is refused: %v", err)
} else if got, err := DecodeForge(data); err != nil || len(got.Trees) != 0 {
t.Errorf("a merged message with no trees comes back as %+v, %v", got, err)
}
for name, dec := range map[string]func([]byte) error{
"forge": func(b []byte) error { _, err := DecodeForge(b); return err },
"group end": func(b []byte) error { _, err := DecodeForgeGroupEnd(b); return err },
} {
for _, in := range []string{`{"group":"g"}`, `{"v":2,"group":"g"}`} {
if err := dec([]byte(in)); !errors.Is(err, delivery.ErrUnknownVersion) {
t.Errorf("%s decode of %s = %v, want delivery.ErrUnknownVersion", name, in, err)
}
}
}
})
t.Run("refused", func(t *testing.T) {
at := func(i int, mut func(*Forge)) Forge { f := sampleForges()[i]; mut(&f); return f }
in := func(mut func(*Forge)) Forge { f := mergedWithInputs(); mut(&f); return f }
pullIn := func(mut func(*Forge)) Forge {
f := Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened,
Dirs: []string{"a"}, DirsKnown: true, Manifests: map[string]string{"a": oddManifest},
Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("0", 64)}}}
mut(&f)
return f
}
for _, c := range []struct {
name, want string
f Forge
}{
{"merged without merge commit", "merge", at(1, func(f *Forge) { f.Merge = "" })},
{"merged without root", "root", at(1, func(f *Forge) { f.Root = "" })},
{"merged with a manifest and no trees", "modules/a", at(1, func(f *Forge) { f.Trees = nil; f.Manifests = map[string]string{"modules/a": "{}"} })},
{"merged with an empty tree", "modules/a", at(1, func(f *Forge) { f.Trees = map[string]string{"modules/a": ""} })},
{"pull with an unknown action", "reopened", at(0, func(f *Forge) { f.Action = "reopened" })},
{"pull without head", "head", at(0, func(f *Forge) { f.Head = "" })},
{"refused without words", "words", at(2, func(f *Forge) { f.Words = "" })},
{"refused without act", "act", at(2, func(f *Forge) { f.Act = "" })},
{"unknown kind", "closed", Forge{Kind: "closed", Repository: "a/b"}},
{"bad repository", "nope", Forge{Kind: ForgePull, Repository: "nope", Number: 1, Head: "h", Action: "opened"}},
{"open with a repository", "only the open", Forge{Kind: ForgeOpen, Repository: "a/b", Open: []OpenPull{{Repository: "a/b", Number: 1}}}},
{"open with a bad entry", "", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a", Number: 1}}}},
{"open with a zero number", "no number", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a/b"}}}},
{"pull naming open pull requests", "only an open message", Forge{Kind: ForgePull, Repository: "a/b", Number: 1, Head: "h", Action: "updated", Open: []OpenPull{{Repository: "a/b", Number: 1}}}},
{"manifest outside trees", `"zz"`, in(func(f *Forge) { f.Manifests["zz"] = "{}" })},
{"locks outside trees", `"yy"`, in(func(f *Forge) { f.Locks["yy"] = map[string]string{"go.sum": "h"} })},
{"empty manifest", `"b"`, in(func(f *Forge) { f.Manifests["b"] = "" })},
{"empty lock hash", `"a"`, in(func(f *Forge) { f.Locks["a"]["go.sum"] = "" })},
{"named paths outside trees", `"other"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"other": {"src": "o"}} })},
{"a named path with no object", `"src"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"a": {"src": ""}} })},
{"placement files", "mesh-placement", in(func(f *Forge) { f.RetiredPlacements = []byte(`{"anchor":"{}"}`) })},
{"placements read", "mesh-placement", in(func(f *Forge) { f.RetiredRead = true })},
{"predefined not an object", "one JSON object", in(func(f *Forge) { f.Predefined = []byte(`["domain"]`) })},
{"predefined both", "says the merge holds none", in(func(f *Forge) { f.Predefined = []byte(`{}`); f.PredefinedRead = true })},
{"pull with the predefined list", "only a merged message", Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened, PredefinedRead: true}},
{"pull manifest outside dirs", `"b"`, pullIn(func(f *Forge) { f.Manifests["b"] = "{}" })},
{"pull locks outside dirs", `"c"`, pullIn(func(f *Forge) { f.Locks["c"] = map[string]string{"go.sum": "h"} })},
{"refused with manifests", "no manifests", Forge{Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Manifests: map[string]string{"a": "{}"}}},
{"open with locks", "only the open", Forge{Kind: ForgeOpen, Locks: map[string]map[string]string{"a": {"go.sum": "h"}}}},
// 1585/T1: a group only on a merged message, with its members, the message's own pull request among them.
{"pull with a group", "only a merged message names its delivery group", at(0, func(f *Forge) {
f.Group, f.Members = "g", []delivery.Head{{Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa"}}
})},
{"merged with a group and no members", "together", at(4, func(f *Forge) { f.Members = nil })},
{"merged with members and no group", "together", at(4, func(f *Forge) { f.Group = "" })},
{"merged with members leaving itself out", "leave out", at(4, func(f *Forge) { f.Members = f.Members[1:] })},
{"merged with a member twice", "twice", at(4, func(f *Forge) { f.Members = append(f.Members, f.Members[0]) })},
{"open with a group", "only the open pull requests", Forge{Kind: ForgeOpen, Group: "g"}},
} {
if _, err := EncodeForge(c.f); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want)
}
}
// A member's repository in another case still names the merged pull request (10c91fba).
f := sampleForges()[4]
f.Members[0].Repository = "Novox/Mesh-SDK"
if _, err := EncodeForge(f); err != nil {
t.Errorf("a member in another case: %v", err)
}
for _, c := range []struct {
name, want string
e ForgeGroupEnd
}{
{"group end without members", "no member", ForgeGroupEnd{Group: "g"}},
{"group end without a group", "names its group", ForgeGroupEnd{Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}}}},
{"group end with a member without its merge", "merge commit", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1}}}},
{"group end with a member twice", "twice", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}, {Repository: "a/b", Number: 1, Merge: "m"}}}},
} {
if _, err := EncodeForgeGroupEnd(c.e); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want)
}
}
})
}
@@ -1,5 +0,0 @@
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg
go 1.26.0
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
@@ -1,14 +0,0 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
@@ -1,78 +0,0 @@
package forgemsg
import (
"encoding/json"
"fmt"
"reflect"
)
// ForgeGroupEndEvent is the name of a delivery group's end on the forge bridge's stream (novox/hq ADR 0633 §4): the
// bridge publishes it once every approved member of the group is merged, after the last member's merged message. It
// has three parts, so no repository's forge event (`forge.<owner>.<name>.<kind>`) is ever this name.
const ForgeGroupEndEvent = "forge.group.merged"
// GroupMerge is one merged member of a delivery group: the repository as `<owner>/<name>`, its pull request and the
// merge commit the forge made of it.
type GroupMerge struct {
Repository string `json:"repository"`
Number int `json:"number"`
Merge string `json:"merge"`
}
// ForgeGroupEnd is the body of ForgeGroupEndEvent: the group and every member merged in it, with its merge commit.
// The indexer takes the members it holds in as one source revision once it hears it (ADR 0633 §4).
type ForgeGroupEnd struct {
V int `json:"v"`
Group string `json:"group"`
Members []GroupMerge `json:"members"`
}
func (e ForgeGroupEnd) check() error {
if e.Group == "" {
return fmt.Errorf("delivery: a group's end names its group")
}
if len(e.Members) == 0 {
return fmt.Errorf("delivery: the end of the group %q names no member merged in it", e.Group)
}
seen := map[string]bool{}
for _, m := range e.Members {
if err := checkRepository(m.Repository); err != nil {
return err
}
if m.Number <= 0 || m.Merge == "" {
return fmt.Errorf("delivery: the member %s of the group %q's end needs its pull request number and merge commit", m.Repository, e.Group)
}
k := fmt.Sprintf("%s#%d", m.Repository, m.Number)
if seen[k] {
return fmt.Errorf("delivery: the end of the group %q names its member %s twice", e.Group, k)
}
seen[k] = true
}
return nil
}
// EncodeForgeGroupEnd is e as JSON at Version; it refuses an end with no group or no member, and a member without its
// pull request or merge commit.
func EncodeForgeGroupEnd(e ForgeGroupEnd) ([]byte, error) {
if err := e.check(); err != nil {
return nil, err
}
e.V = Version
return json.Marshal(e)
}
// DecodeForgeGroupEnd reads a group's end; it refuses an unknown version (ErrUnknownVersion) and what
// EncodeForgeGroupEnd refuses.
func DecodeForgeGroupEnd(data []byte) (ForgeGroupEnd, error) {
var e ForgeGroupEnd
if err := decode(data, &e); err != nil {
return ForgeGroupEnd{}, err
}
if err := e.check(); err != nil {
return ForgeGroupEnd{}, err
}
return e, nil
}
// SameForgeGroupEnd says whether two group ends are the same value.
func SameForgeGroupEnd(a, b ForgeGroupEnd) bool { return reflect.DeepEqual(a, b) }
+1 -4
View File
@@ -3,13 +3,12 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
@@ -21,5 +20,3 @@ require (
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ./forgemsg
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2 h1:MOuutmbKBL1GtTZU9W1+ja/K7TL05EFU6hEt3Swf/oE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -5,7 +5,6 @@ import (
"slices"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -146,11 +145,11 @@ func MergeActs(value []byte) ([]Act, error) {
// RefusedMessage is the refused message the bridge writes on FORGE when the forge refuses an act: the act, the pull
// request and head it was on, and the forge's own words.
func RefusedMessage(a Act, words string) (Message, error) {
event, err := forgemsg.ForgeEvent(a.Repository, forgemsg.ForgeRefused)
event, err := delivery.ForgeEvent(a.Repository, delivery.ForgeRefused)
if err != nil {
return Message{}, err
}
body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeRefused, Repository: a.Repository, Number: a.Number,
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeRefused, Repository: a.Repository, Number: a.Number,
Head: a.Head, Act: a.Name, Words: words})
if err != nil {
return Message{}, err
@@ -5,7 +5,6 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -28,7 +27,7 @@ func owedValue(t *testing.T, repository string, number int, act string, o delive
return key, b
}
func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1, K1
func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1
key, v := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate",
delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "pending", Text: "waiting", Target: "https://forge.example/novox/mesh-delivery/pulls/12"})
a, err := OwedAct(key, v)
@@ -43,19 +42,6 @@ func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1, K1
if a.Repository != "novox/mesh-delivery" || a.Number != 12 || a.Head != headA || a.Name != "status=mesh/merge-gate" {
t.Errorf("act = %+v", a)
}
// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no
// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge
// performs no mesh/check act.
ck, cv := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"})
if a, err := OwedAct(ck, cv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err)
}
// Nor through the gate's own owed bucket: the context is refused whichever key carries it.
gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check",
delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"})
if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err)
}
}
// checkerKV is the status the checker owes, as it writes it (mesh-sdk go/delivery CheckerOwedKey, EncodeCheckerOwed).
@@ -72,6 +58,23 @@ func checkerKV(t *testing.T, o delivery.Owed) (string, []byte) {
return key, v
}
// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no
// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge
// performs no mesh/check act.
func TestOwedMeshCheckIsNoToolCall(t *testing.T) { // K1
key, v := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"})
a, err := OwedAct(key, v)
if err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err)
}
// Nor through the gate's own owed bucket: the context is refused whichever key carries it.
gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check",
delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"})
if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err)
}
}
func TestOwedViewNoteAndCommentCalls(t *testing.T) { // 1341/AC2
cases := []struct {
act string
@@ -108,12 +111,18 @@ func TestOwedActWithoutAPullRequestNeedsOneOnlyWhereTheToolDoes(t *testing.T) {
}
}
func TestOwedActThatDisagreesWithItsKeyIsRefused(t *testing.T) {
// A dotted repository's key is unescaped, and its act names the repository whole.
dk, dv := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"})
if a, err := OwedAct(dk, dv); err != nil || a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" {
t.Errorf("a dotted repository's act = %+v, %v", a, err)
func TestOwedActOfADottedRepositoryIsUnescaped(t *testing.T) {
key, v := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"})
a, err := OwedAct(key, v)
if err != nil {
t.Fatal(err)
}
if a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" {
t.Errorf("act = %+v", a)
}
}
func TestOwedActThatDisagreesWithItsKeyIsRefused(t *testing.T) {
key, _ := owedValue(t, "novox/mesh-delivery", 12, "view", delivery.Owed{Act: delivery.OwedView, Text: "v"})
v, _ := delivery.EncodeOwed(delivery.Owed{Act: delivery.OwedComment, Text: "v"})
if _, err := OwedAct(key, v); err == nil {
@@ -157,11 +166,13 @@ func TestMergeActsHandOnlyTheMemberHandedWithEveryApprovedHead(t *testing.T) { /
if acts[0].Name != "merge" || acts[0].Number != 12 || acts[0].Head != headA {
t.Errorf("act = %+v", acts[0])
}
// A merge with no approved heads is refused: the forge's tool would refuse it with nothing to say which.
v = mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{
}
func TestMergeActsOfAMergeWithoutApprovedHeadsAreRefused(t *testing.T) {
v := mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{
{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, State: delivery.MergeMerging}}})
if _, err := MergeActs(v); err == nil {
t.Error("a merge with no approved heads was taken")
t.Error("a merge with no approved heads was taken: the forge's tool would refuse it with nothing to say which")
}
}
@@ -174,13 +185,13 @@ func TestRefusedMessageNamesTheActAndTheForgesWords(t *testing.T) { // 1341/AC3,
if m.Event != "forge.novox.mesh-delivery.refused" {
t.Errorf("event = %q", m.Event)
}
f, err := forgemsg.DecodeForge(m.Data)
f, err := delivery.DecodeForge(m.Data)
if err != nil {
t.Fatal(err)
}
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA,
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA,
Act: "merge", Words: "novox/mesh-delivery#12 is not merged: its head moved"}
if !forgemsg.SameForge(f, want) {
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v", f)
}
}
@@ -15,7 +15,7 @@ import (
"strings"
"sync"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ErrUnreadable is wrapped by every error that says an event cannot be turned into a message; it is never
@@ -29,6 +29,9 @@ var ErrNotHandled = errors.New("event not handled")
// heard again later.
var ErrRetry = errors.New("the forge could not say")
// ErrNoModules says a merge into the trunk left a repository with no module.json anywhere: no merged message.
var ErrNoModules = errors.New("no module.json in the merge")
// Trunk is the branch a merged message is written for.
const Trunk = "main"
@@ -97,9 +100,7 @@ func unreadable(subject, why string, args ...any) error {
// Convert turns one event of the forge module, by its bus subject and body, into a message. An event it cannot read
// is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
// pull request left its group. A forge that cannot say is ErrRetry. A merge that leaves no module.json anywhere is a merged message with no
// trees all the same (issue 1126): the indexer takes it in and drops the repository's modules, and the view reads the
// repository as reached at it.
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
return ConvertKept(subject, data, trees, files, nil)
}
@@ -110,11 +111,11 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
var action string
switch subject {
case UpdatedSubject:
action = forgemsg.PullUpdated
action = delivery.PullUpdated
case ClosedSubject:
action = forgemsg.PullClosed
action = delivery.PullClosed
case MergedSubject:
action = forgemsg.PullClosed // unless it merged into the trunk, below
action = delivery.PullClosed // unless it merged into the trunk, below
default:
return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject)
}
@@ -136,29 +137,29 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
return merged(subject, p, trees, files, kept)
}
said := ""
if action == forgemsg.PullUpdated && p.Base != "" && p.Base != Trunk {
if action == delivery.PullUpdated && p.Base != "" && p.Base != Trunk {
// The checker judges pull requests into main only, and source.main names main's head only (novox/hq ADR 0484 §11): a
// pull request into another base is no candidate, as it is none at the start (OpenPulls) and at its merge. It is
// written closed, so a candidate an older head made is taken out, and said (issue 1422).
action = forgemsg.PullClosed
action = delivery.PullClosed
said = fmt.Sprintf("%s/%s#%d at %s is into %s, not %s: written closed, so it is no candidate; the checker judges pull requests into %s only (novox/hq ADR 0484 §11)",
p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base, Trunk, Trunk)
}
f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Action: action}
// The directories are the forge's word only when it said it could read them (module_dirs_said).
if action == forgemsg.PullUpdated && p.ModuleDirsSaid {
if action == delivery.PullUpdated && p.ModuleDirsSaid {
f.Dirs, f.DirsKnown = p.ModuleDirs, true
var err error
if f.Manifests, f.Locks, _, err = inputsRead(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
}
}
event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
body, err := forgemsg.EncodeForge(f)
body, err := delivery.EncodeForge(f)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
@@ -293,6 +294,9 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
if err != nil {
return Message{}, fmt.Errorf("%w: %s: the trees of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
}
if len(dirs) == 0 {
return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s", ErrNoModules, subject, repository, p.MergeSHA)
}
names := make([]string, 0, len(dirs))
for dir := range dirs {
names = append(names, dir)
@@ -313,11 +317,11 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
if len(locks) == 0 {
locks = nil
}
event, err := forgemsg.ForgeEvent(repository, forgemsg.ForgeMerged)
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
f := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
if repository == CatalogRepository {
// The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config.
@@ -331,15 +335,11 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
f.Predefined = json.RawMessage(raw)
}
}
body, err := forgemsg.EncodeForge(f)
body, err := delivery.EncodeForge(f)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
msg := Message{Event: event, Data: body}
if len(dirs) == 0 {
msg.Said = fmt.Sprintf("%s at the merge commit %s holds no module.json: a merged message with no module directories", repository, p.MergeSHA)
}
return msg, nil
return Message{Event: event, Data: body}, nil
}
// openPull is an entry of gitea_open_pulls's answer (mesh-catalog modules/gitea/client.ts:369-381 openPulls, with the
@@ -373,12 +373,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
skipped = append(skipped, name+": it names no owner, repo, number, branch or 40-hex head")
continue
}
f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Action: forgemsg.PullUpdated}
event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Action: delivery.PullUpdated}
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
var body []byte
if err == nil {
body, err = forgemsg.EncodeForge(f)
body, err = delivery.EncodeForge(f)
}
if err != nil {
skipped = append(skipped, name+": "+err.Error())
@@ -392,12 +392,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
// Track says what a message the bridge wrote changes in the head it keeps for each pull request (the bucket heads,
// novox/hq ADR 0443 §5): the key `<owner>.<name>.<number>`, and the head to keep under it, or "" to take the key away
// (a closed pull request, a merged one). ok is false for a message that keeps no head.
func Track(f forgemsg.Forge) (key, head string, ok bool) {
func Track(f delivery.Forge) (key, head string, ok bool) {
key = strings.ReplaceAll(f.Repository, "/", ".") + "." + strconv.Itoa(f.Number)
switch {
case f.Kind == forgemsg.ForgePull && f.Action != forgemsg.PullClosed:
case f.Kind == delivery.ForgePull && f.Action != delivery.PullClosed:
return key, f.Head, true
case f.Kind == forgemsg.ForgePull || f.Kind == forgemsg.ForgeMerged:
case f.Kind == delivery.ForgePull || f.Kind == delivery.ForgeMerged:
return key, "", true
}
return "", "", false
@@ -413,22 +413,22 @@ func OpenMessage(answer []byte) (Message, error) {
if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil {
return Message{}, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err)
}
open := []forgemsg.OpenPull{}
open := []delivery.OpenPull{}
for _, p := range *a.Pulls {
if p.Base != Trunk || p.Owner == "" || p.Repo == "" || p.Number <= 0 {
continue
}
open = append(open, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
open = append(open, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
}
slices.SortFunc(open, func(a, b forgemsg.OpenPull) int {
slices.SortFunc(open, func(a, b delivery.OpenPull) int {
if c := strings.Compare(a.Repository, b.Repository); c != 0 {
return c
}
return a.Number - b.Number
})
body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeOpen, Open: open})
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeOpen, Open: open})
if err != nil {
return Message{}, fmt.Errorf("the open pull requests cannot be said: %w", err)
}
return Message{Event: forgemsg.OpenEvent(), Data: body}, nil
return Message{Event: delivery.OpenEvent(), Data: body}, nil
}
@@ -6,7 +6,6 @@ import (
"strings"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -55,13 +54,13 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion;
if msg.Event != "forge.novox.mesh-delivery.pull" {
t.Errorf("event = %q", msg.Event)
}
f, err := forgemsg.DecodeForge(msg.Data)
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
if !forgemsg.SameForge(f, want) {
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
}
@@ -71,7 +70,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
if err != nil {
t.Fatal(err)
}
f, _ := forgemsg.DecodeForge(msg.Data)
f, _ := delivery.DecodeForge(msg.Data)
if f.DirsKnown || len(f.Dirs) != 0 {
t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown)
}
@@ -80,7 +79,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
if err != nil {
t.Fatal(err)
}
f, _ = forgemsg.DecodeForge(msg.Data)
f, _ = delivery.DecodeForge(msg.Data)
if !f.DirsKnown || len(f.Dirs) != 0 {
t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown)
}
@@ -89,7 +88,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
if err != nil {
t.Fatal(err)
}
if f, _ = forgemsg.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 {
if f, _ = delivery.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 {
t.Errorf("dirs %v known %v without module_dirs_said", f.Dirs, f.DirsKnown)
}
}
@@ -102,13 +101,13 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
if msg.Event != "forge.novox.mesh-delivery.pull" {
t.Errorf("event = %q", msg.Event)
}
f, err := forgemsg.DecodeForge(msg.Data)
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: forgemsg.PullClosed}
if !forgemsg.SameForge(f, want) {
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: delivery.PullClosed}
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
// Issue 1422, ADR 0484 §11: a head pushed to a pull request into another base is written closed, read nothing for, and said;
@@ -117,12 +116,12 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
if err != nil {
t.Fatal(err)
}
if f, err = forgemsg.DecodeForge(msg.Data); err != nil || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
if f, err = delivery.DecodeForge(msg.Data); err != nil || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
t.Errorf("an update into nox-mesh is %+v (%v), said %q; want it closed and said", f, err, msg.Said)
}
for _, base := range []any{"main", nil} {
msg, err = Convert(updatedSubject, updated(map[string]any{"base": base}), nil, anyFiles)
if f, _ = forgemsg.DecodeForge(msg.Data); err != nil || f.Action != forgemsg.PullUpdated || msg.Said != "" {
if f, _ = delivery.DecodeForge(msg.Data); err != nil || f.Action != delivery.PullUpdated || msg.Said != "" {
t.Errorf("an update with base %v is %+v (%v), said %q; want it updated", base, f, err, msg.Said)
}
}
@@ -171,18 +170,12 @@ func mustDrop(body []byte, key string) []byte {
return b
}
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235
for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} {
if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) {
t.Errorf("%s: err = %v, want ErrNotHandled", s, err)
}
}
// 1236: a merged event without a full merge commit is unreadable, and the forge is not asked.
var calls []call
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
t.Errorf("merged without a merge commit: err = %v calls = %v", err, calls)
}
}
const merge = "89abcdef0123456789abcdef0123456789abcdef"
@@ -229,13 +222,13 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
if msg.Event != "forge.novox.mesh-delivery.merged" {
t.Errorf("event = %q", msg.Event)
}
f, err := forgemsg.DecodeForge(msg.Data)
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}}
if !forgemsg.SameForge(f, want) {
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
if len(calls) != 1 || calls[0] != (call{"novox", "mesh-delivery", merge}) {
@@ -249,13 +242,13 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
if err != nil {
t.Fatal(err)
}
f, err := forgemsg.DecodeForge(msg.Data)
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: forgemsg.PullClosed}
if msg.Event != "forge.novox.mesh-delivery.pull" || !forgemsg.SameForge(f, want) {
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: delivery.PullClosed}
if msg.Event != "forge.novox.mesh-delivery.pull" || !delivery.SameForge(f, want) {
t.Errorf("event %q message %+v, want a closed pull message", msg.Event, f)
}
if len(calls) != 0 {
@@ -276,19 +269,19 @@ func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the co
}
}
// 1126 (note 94): a merge that leaves no module.json anywhere (hq, or a repository whose last module was removed) is a merged
// message with no trees, so the indexer takes it in and the repository's reach is not held for ever.
func TestNoModuleJSONAnywhereWritesAMergedMessageWithNoTrees(t *testing.T) {
func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (notes: a repository with no module.json)
var calls []call
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil), anyFiles)
if err != nil {
t.Fatal(err)
if !errors.Is(err, ErrNoModules) || msg.Data != nil {
t.Errorf("err = %v data = %s", err, msg.Data)
}
f, err := forgemsg.DecodeForge(msg.Data)
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Merge: merge, Root: "r0"}
if err != nil || msg.Event != "forge.novox.mesh-delivery.merged" || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "no module.json") {
t.Errorf("event %q message %+v said %q, %v; want a merged message with no trees", msg.Event, f, msg.Said, err)
}
func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236
var calls []call
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
t.Errorf("err = %v calls = %v", err, calls)
}
}
@@ -310,12 +303,12 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/
if err != nil || len(skipped) != 0 || len(msgs) != 2 {
t.Fatalf("msgs %d skipped %v err %v", len(msgs), skipped, err)
}
for i, want := range []forgemsg.Forge{
{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated},
{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated},
for i, want := range []delivery.Forge{
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated},
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated},
} {
f, err := forgemsg.DecodeForge(msgs[i].Data)
if err != nil || !forgemsg.SameForge(f, want) {
f, err := delivery.DecodeForge(msgs[i].Data)
if err != nil || !delivery.SameForge(f, want) {
t.Errorf("message %d = %+v err %v, want %+v", i, f, err, want)
}
if f.DirsKnown {
@@ -327,11 +320,14 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/
}
}
func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237, 1237/AC4
// 1237/AC4: an open pull request into another branch gets no message.
if msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head))); err != nil || len(msgs) != 0 || len(skipped) != 0 {
t.Errorf("into another branch: msgs %d skipped %v err %v, want none", len(msgs), skipped, err)
func TestOpenPullIntoAnotherBranchGetsNoMessage(t *testing.T) { // 1237/AC4
msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head)))
if err != nil || len(msgs) != 0 || len(skipped) != 0 {
t.Errorf("msgs %d skipped %v err %v, want none", len(msgs), skipped, err)
}
}
func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237
msgs, skipped, err := OpenPulls(openAnswer(openEntry("a", 1, "main", "b", "short"), openEntry("mesh-delivery", 12, "main", "feat/x", head)))
if err != nil || len(msgs) != 1 || len(skipped) != 1 || !strings.Contains(skipped[0], "novox/a#1") {
t.Errorf("msgs %d skipped %v err %v", len(msgs), skipped, err)
@@ -351,15 +347,15 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6
if err != nil || m.Event != "forge.open" {
t.Fatalf("message %+v err %v", m, err)
}
f, err := forgemsg.DecodeForge(m.Data)
f, err := delivery.DecodeForge(m.Data)
// Every one into the trunk, whether its head reads or not (it is open); none into another branch; in a fixed order.
want := []forgemsg.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
if err != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] {
want := []delivery.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
if err != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] {
t.Errorf("open message = %+v err %v, want %v", f, err, want)
}
// Nothing open is a message too: the indexer then drops every candidate.
m, err = OpenMessage([]byte(`{"pulls":[]}`))
if f, derr := forgemsg.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 0 {
if f, derr := delivery.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 0 {
t.Errorf("empty answer: %+v %v %v", f, err, derr)
}
for _, bad := range []string{`{"pulls":null}`, `{}`, `x`} {
@@ -370,24 +366,24 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6
}
func TestTrackSaysWhichHeadOfWhichPullRequestWasWritten(t *testing.T) { // 1279/AC6
pullMsg := func(action string) forgemsg.Forge {
return forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action}
pullMsg := func(action string) delivery.Forge {
return delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action}
}
for _, a := range []string{forgemsg.PullOpened, forgemsg.PullUpdated} {
for _, a := range []string{delivery.PullOpened, delivery.PullUpdated} {
if key, h, ok := Track(pullMsg(a)); key != "novox.mesh-delivery.12" || h != head || !ok {
t.Errorf("%s: Track = %q %q %v, want the head kept under a bucket key", a, key, h, ok)
}
}
// A closed pull request and a merged one are no longer kept: their key goes.
if key, h, ok := Track(pullMsg(forgemsg.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok {
if key, h, ok := Track(pullMsg(delivery.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok {
t.Errorf("closed: Track = %q %q %v", key, h, ok)
}
merged := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}
merged := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}
if key, h, ok := Track(merged); key != "novox.mesh-delivery.12" || h != "" || !ok {
t.Errorf("merged: Track = %q %q %v", key, h, ok)
}
// The open message and the rest hold no head.
if _, _, ok := Track(forgemsg.Forge{Kind: forgemsg.ForgeOpen}); ok {
if _, _, ok := Track(delivery.Forge{Kind: delivery.ForgeOpen}); ok {
t.Error("an open message was tracked")
}
}
@@ -1,79 +0,0 @@
package bridge
import (
"context"
"encoding/json"
"fmt"
"regexp"
)
// Caller calls a module's tool by its manifest key (`<module>.<tool>`) with a JSON body and returns the tool's result:
// the bridge's process calls through the bus as its own principal, its tools bundle through the runtime (stdio.Ask).
type Caller func(ctx context.Context, key string, body any) (json.RawMessage, error)
// CommitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner,
// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4).
const CommitTreesTool = "gitea.gitea_commit_trees"
// TreesVia asks the forge for a commit's trees through call.
func TreesVia(ctx context.Context, call Caller) Trees {
return func(owner, repo, sha string) (string, map[string]string, error) {
raw, err := call(ctx, CommitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha})
if err != nil {
return "", nil, err
}
var a struct {
Root string `json:"root"`
Trees map[string]string `json:"trees"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", CommitTreesTool, err)
}
if a.Root == "" {
return "", nil, fmt.Errorf("the answer of %s names no root tree", CommitTreesTool)
}
return a.Root, a.Trees, nil
}
}
// GetFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
const GetFileTool = "gitea.gitea_get_file"
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
// recognises it as /: 404 /).
var notFound = regexp.MustCompile(`: 404 `)
// FilesVia asks the forge for a file through call. A file the forge says is not there is ErrNotFound.
func FilesVia(ctx context.Context, call Caller) Files {
return func(owner, repo, path, ref string) ([]byte, error) {
raw, err := call(ctx, GetFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
if err != nil {
if notFound.MatchString(err.Error()) {
return nil, fmt.Errorf("%w: %v", ErrNotFound, err)
}
return nil, err
}
var a struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", GetFileTool, err)
}
return []byte(*a.File.Content), nil
}
}
// MergedMessage is the merged message of a merge into the trunk, asked again: built as the bridge builds the one it writes
// on FORGE (it is that event's body through ConvertKept), from the pull request's owner, repository, number, branch, head
// and merge commit, which the gate's merges state holds. It reads the forge for every module directory (no kept inputs).
func MergedMessage(owner, repo string, number int, branch, head, merge string, trees Trees, files Files) (Message, error) {
body, err := json.Marshal(pull{Owner: owner, Repo: repo, Number: number, Head: branch, HeadSHA: head, Base: Trunk, MergeSHA: merge})
if err != nil {
return Message{}, err
}
return ConvertKept(MergedSubject, body, trees, files, nil)
}
@@ -13,7 +13,7 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the
@@ -69,7 +69,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
if err != nil {
t.Fatal(err)
}
f, err := forgemsg.DecodeForge(msg.Data)
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
@@ -90,7 +90,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
if err != nil {
t.Fatal(err)
}
if f, err = forgemsg.DecodeForge(msg.Data); err != nil {
if f, err = delivery.DecodeForge(msg.Data); err != nil {
t.Fatal(err)
}
if len(f.RetiredPlacements) > 0 || f.RetiredRead || strings.Contains(string(msg.Data), `"placements`) {
@@ -108,7 +108,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
}
forge.have[PredefinedFile] = []byte(`{"font-size":{"type":"integer","value":13}}`)
msg, _ = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get)
if f, err = forgemsg.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead {
if f, err = delivery.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead {
t.Errorf("predefined = %s (%v)", f.Predefined, err)
}
forge.calls = nil
@@ -133,7 +133,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
if err != nil {
t.Fatal(err)
}
f, _ := forgemsg.DecodeForge(msg.Data)
f, _ := delivery.DecodeForge(msg.Data)
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 1 || len(f.Packages) != 0 {
t.Errorf("manifests %v locks %v packages %v", f.Manifests, f.Locks, f.Packages)
}
@@ -146,7 +146,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
if err != nil {
t.Fatal(err)
}
f, _ = forgemsg.DecodeForge(msg.Data)
f, _ = delivery.DecodeForge(msg.Data)
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
}
@@ -175,7 +175,7 @@ func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testin
if err != nil {
t.Fatal(err)
}
if f, _ := forgemsg.DecodeForge(msg.Data); f.Locks != nil {
if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil {
t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks)
}
}
@@ -4,7 +4,7 @@ import (
"os"
"testing"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ADR 0495 clause 1 with the lead's decision L8 of R4e: the forge bridge is no reader of a manifest's `paths`. A merge of a
@@ -26,7 +26,7 @@ func TestAMergedMessageCarriesNoNamedPathsTheBridgeDerived(t *testing.T) {
if err != nil {
t.Fatal(err)
}
f, err := forgemsg.DecodeForge(msg.Data)
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
@@ -1,198 +0,0 @@
// Package groups is what the forge bridge's process and its tools bundle share of a delivery group's end (novox/hq ADR
// 0633 §4, issue 1585): the keys of the bridge's state performed, and the asks of the state end-asks, through which the
// tool bridge_group_end_again asks the process to say a group's end again (the bridge serves no request subject, ADR 0437
// §4, so a tool asks in a state the process watches, as indexer_take_in asks the indexer in takeins).
package groups
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The bridge's states the tools read and write.
const (
// PerformedState is the bridge's state of what it performed and published.
PerformedState = "performed"
// EndAsksState is the state of asks to say a group's end again, keyed by the gate's merges key of the group.
EndAsksState = "end-asks"
// EndAsksBucket is its bucket on the bus.
EndAsksBucket = delivery.ForgeBridgeModule + "_" + EndAsksState
)
// Key is the key of performed that says a group's end is written: `groups.<merges key>`, its value the revision of the
// gate's merges key it was worked out from.
func Key(mergesKey string) string { return "groups." + mergesKey }
// PublishedKey is the key of performed that says a member's merged message is written: `published.<16 hex digits of the
// sha256 of <repository>#<number>@<merge>>`.
func PublishedKey(repository string, number int, merge string) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge)))
return "published." + hex.EncodeToString(sum[:8])
}
// EndID is the message id of a group's end, and of one said again by hand at a time: the stream's duplicate window drops a
// second publish of the same id.
func EndID(group, lastMerge, again string) string {
id := "group-end." + group + "." + lastMerge
if again != "" {
id += "." + again
}
return id
}
// AgainStamp is the time a hand ask appends to a group end's message id.
func AgainStamp(at time.Time) string { return at.UTC().Format("20060102T150405.000000000Z") }
// EndAsk is a value of the end-asks state: which group's end a person asked to be said again, why, who and when, and, once
// the bridge read it, its answer.
type EndAsk struct {
V int `json:"v"`
Group string `json:"group"`
Why string `json:"why,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Written bool `json:"written,omitempty"`
Answer string `json:"answer,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// Done says the ask is answered.
func (a EndAsk) Done() bool { return !a.Answered.IsZero() }
// NewEndAsk is the value bridge_group_end_again writes for a group: its key (the gate's merges key) and the value.
func NewEndAsk(group, why, by string, at time.Time) (string, []byte, error) {
group = strings.TrimSpace(group)
if group == "" {
return "", nil, errors.New("name the group whose end is said again (group)")
}
key, err := delivery.MergesKey(group)
if err != nil {
return "", nil, err
}
data, err := json.Marshal(EndAsk{V: 1, Group: group, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
return key, data, err
}
// DecodeEndAsk reads an end-asks value.
func DecodeEndAsk(data []byte) (EndAsk, error) {
var a EndAsk
if err := json.Unmarshal(data, &a); err != nil {
return a, err
}
if a.V != 1 || a.Group == "" {
return a, errors.New("an end-asks value of no version 1 naming no group")
}
return a, nil
}
// Row is one group as bridge_groups answers it: the gate's merge of it and what the bridge kept of its end.
type Row struct {
Key string `json:"key"`
Group string `json:"group"`
Members []RowMember `json:"members"`
Ended bool `json:"ended"`
EndedAt uint64 `json:"ended-at-revision,omitempty"`
Open bool `json:"open"`
Words string `json:"words"`
// LastEndAsk is the last ask to say the group's end again (bridge_group_end_again), its who, why and when, and the
// bridge's answer once it read it; absent when none was asked. The state end-asks keeps the last per group.
LastEndAsk *EndAsk `json:"last-end-ask,omitempty"`
}
// RowMember is one approved member of a group: its merge's state at the gate and whether its merged message is written.
type RowMember struct {
Repository string `json:"repository"`
Number int `json:"number"`
State string `json:"state"`
Commit string `json:"commit,omitempty"`
Published bool `json:"published"`
}
// Rows are the groups the gate's merges hold with more than one member, each with what the bridge kept: ended (its end
// written, the revision it was worked out from) or open (some member is not merged or its merged message not written, or its
// end not written). merges is the gate's merges by key, performed the bridge's performed keys by key; unread are the gate's
// keys that do not read. asks are the end-asks values by the same key; one that does not read is named unread as
// `end-asks.<key>`.
func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[string]uint64, asks map[string][]byte) (rows []Row, unread []string) {
lastAsk := map[string]*EndAsk{}
for key, raw := range asks {
a, err := DecodeEndAsk(raw)
if err != nil {
unread = append(unread, EndAsksState+"."+key)
continue
}
lastAsk[key] = &a
}
for key, raw := range merges {
m, err := delivery.DecodeMerge(raw)
if err != nil {
unread = append(unread, key)
continue
}
if len(m.Approved) < 2 {
continue
}
r := Row{Key: key, Group: m.Group}
allMerged := true
for _, p := range m.Pulls {
pub := p.Commit != "" && performed[PublishedKey(p.Repository, p.Number, p.Commit)]
r.Members = append(r.Members, RowMember{Repository: p.Repository, Number: p.Number, State: string(p.State), Commit: p.Commit, Published: pub})
if p.State != delivery.MergeMerged || !pub {
allMerged = false
}
}
if len(m.Pulls) < len(m.Approved) {
allMerged = false
}
r.EndedAt, r.Ended = endedAt[key]
r.LastEndAsk = lastAsk[key]
r.Open = !r.Ended
switch {
case r.Ended:
r.Words = "its end is written"
case allMerged:
r.Words = "every member is merged and its message written, and its end is not written: bridge_group_end_again says it"
default:
r.Words = "gathered: not every member is merged and its merged message written"
}
rows = append(rows, r)
}
sortRows(rows)
sort.Strings(unread)
return rows, unread
}
func sortRows(rows []Row) { sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) }
// MembersOf are a merge's approved members with their heads, in the order of their keys.
func MembersOf(m delivery.Merge) []delivery.Head {
var out []delivery.Head
for _, k := range slices.Sorted(func(yield func(string) bool) {
for k := range m.Approved {
if !yield(k) {
return
}
}
}) {
at := strings.LastIndex(k, "@")
if at < 0 {
continue
}
n, err := strconv.Atoi(k[at+1:])
if err != nil {
continue
}
out = append(out, delivery.Head{Repository: k[:at], Number: n, Head: m.Approved[k]})
}
return out
}
@@ -28,13 +28,12 @@ type state struct {
}
type artifact struct {
Name string `json:"name"`
Kind string `json:"kind"`
Language string `json:"language"`
System string `json:"system"`
From string `json:"from"`
Binary string `json:"binary"`
Loads []string `json:"loads"`
Name string `json:"name"`
Kind string `json:"kind"`
Language string `json:"language"`
System string `json:"system"`
From string `json:"from"`
Binary string `json:"binary"`
}
type manifest struct {
@@ -45,7 +44,6 @@ type manifest struct {
Streams []stream `json:"streams"`
State []state `json:"state"`
Reads []string `json:"reads"`
Tools []string `json:"tools"`
Invokes []string `json:"invokes"`
OwnSecrets map[string]string `json:"own-secrets"`
SecretsOwner string `json:"secrets-owner"`
@@ -109,15 +107,11 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
// §5), the revision of each key of the gate's buckets the bridge performed, so a key is performed once per
// revision (ADR 0437 §8; issue 1341), and what a merge read of each module directory by its tree, so a merge reads
// the forge only for the directories it changed (issue 1126).
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}, {Name: "end-asks", History: 1}}) { // 1279/AC6, 1341, 1126, 1585
t.Errorf("state = %+v, want the buckets heads, performed, inputs and end-asks", m.State)
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}}) { // 1279/AC6, 1341, 1126
t.Errorf("state = %+v, want the buckets heads, performed and inputs", m.State)
}
// 1585/T18: the tools read the groups and ask the bridge to say a group's end again, through its state end-asks. The tool bridge_merged_message answers a merge's merged message again (issue 1585).
if want := []string{"bridge_group_end_again", "bridge_groups", "bridge_merged_message"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// No other state, no bus, no seat, no event (ADR 0437 §4; ADR 0521: no check.status).
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats"} {
// No other state, no bus, no seat, no tool, no event (ADR 0437 §4; ADR 0521: no check.status).
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the bridge has none", k)
}
@@ -149,8 +143,8 @@ func TestManifestProcess(t *testing.T) { // 1235/AC1
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts)
if len(m.Build.Artifacts) != 1 {
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
}
a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) {
+1 -18
View File
@@ -13,13 +13,7 @@
"state": [
{ "name": "heads", "history": 1 },
{ "name": "performed", "history": 1 },
{ "name": "inputs", "history": 1 },
{ "name": "end-asks", "history": 1 }
],
"tools": [
"bridge_groups",
"bridge_group_end_again",
"bridge_merged_message"
{ "name": "inputs", "history": 1 }
],
"reads": [
"mesh-delivery-gate.merges",
@@ -81,17 +75,6 @@
"system": "arch",
"from": "cmd/mesh-delivery-forge-bridge",
"binary": "mesh-delivery-forge-bridge"
},
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-forge-bridge-tools",
"binary": "mesh-delivery-forge-bridge-tools",
"loads": [
"mesh-delivery-forge-bridge-tools"
]
}
]
}
@@ -1,78 +0,0 @@
package main
import (
"fmt"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"mesh-delivery-indexer/internal/index"
)
// catalogRead reads the indexer's modules, manifests and graph states for the seat's verbs.
func catalogRead(modules, manifests, nodes stdio.KeptState) index.CatalogRead {
return index.CatalogRead{
Modules: func() (map[string]delivery.Module, error) {
keys, err := modules.Keys()
if err != nil {
return nil, err
}
out := map[string]delivery.Module{}
for _, k := range keys {
e, err := modules.Get(k)
if err != nil {
return nil, fmt.Errorf("modules.%s: %w", k, err)
}
if e == nil {
continue
}
if m, err := delivery.DecodeModule(e.Value); err == nil {
out[k] = m
}
}
return out, nil
},
Manifest: func(digest string) (string, error) {
key, err := delivery.ManifestsKey(digest)
if err != nil {
return "", err
}
e, err := manifests.Get(key)
if err != nil {
return "", err
}
if e == nil {
return "", fmt.Errorf("the manifests state holds no %s", digest)
}
kept, err := delivery.DecodeManifest(e.Value)
return kept.JSON, err
},
Graph: func() (graph.Graph, error) {
keys, err := nodes.Keys()
if err != nil {
return nil, err
}
g := graph.Graph{}
for _, k := range keys {
e, err := nodes.Get(k)
if err != nil {
return nil, fmt.Errorf("graph.%s: %w", k, err)
}
if e == nil {
continue
}
if n, err := graph.Decode(e.Value); err == nil {
g[k] = n
}
}
return g, nil
},
}
}
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
func catalogTools() []stdio.Tool {
return index.CatalogTools(catalogRead(stdio.State("modules"), stdio.State("manifests"), stdio.State(graph.State)))
}
@@ -8,7 +8,6 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"strings"
@@ -23,23 +22,10 @@ import (
// asker writes a takeins value and waits for the indexer's answer to it.
type asker func(key string, value []byte) (index.TakeInAsk, error)
// Tools are the indexer's tools. read is the indexer's state of verdicts, which indexer_verdicts reads; counts the old sections' count.
func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() (index.OldSectionsCount, error)) []stdio.Tool {
// Tools are the indexer's tools. read is the indexer's state of verdicts, which indexer_verdicts reads.
func Tools(ask asker, now func() time.Time, read stdio.KeptState) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
return []stdio.Tool{
{Name: index.OldSectionsTool,
Description: "How many accepted modules use each old section of a manifest (state, logs, streams, objects), the foundation " +
"(nats, mesh-grants, the indexer) left out: the count the judge refuses a use of a section at 0 by, as modules move " +
"their bus state into `requires` by template. Read from the indexer's oldsections state, kept as the indexer judges " +
"the manifests; an empty answer means the indexer has not judged since it started.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
c, err := counts()
if err != nil {
return nil, err
}
return c.Counts, nil
}},
{Name: index.TakeInTool,
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
@@ -49,24 +35,12 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
"indexer) is read again at that head and written when it changed. Answers stored, from which " +
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
"indexer_module (mesh-delivery-view) reads the result. With repository instead of module it takes in the head the gate's " +
"source names for that repository, for a merge whose merged message the indexer never heard (the forge bridge has no " +
"tool): a head whose trunk holds no module.json is taken in as a merge that leaves none (every module of the repository " +
"is dropped, one another repository's trunk holds handed over from there first, listed in handed and dropped); a head " +
"that holds modules is answered \"take each module in by name\".",
Input: map[string]any{"type": "object", "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
"repository": map[string]any{"type": "string", "description": "instead of module: the repository (owner/name) whose head is taken in"},
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
"indexer_module (mesh-delivery-view) reads the result.",
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
var key string
var value []byte
var err error
if str(args, "repository") != "" && str(args, "module") == "" {
key, value, err = index.NewTakeInRepository(str(args, "repository"), str(args, "why"), stdio.CallerOf(args), now())
} else {
key, value, err = index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
}
key, value, err := index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
@@ -105,14 +79,12 @@ func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func()
Description: "Judge the manifests again now, as the indexer does at its start and after each merge, and answer how the module's " +
"verdict stands (waits, refused or accepted, and why). For a manifest that waits for a judge that knows its fields, or " +
"after a manifest was mended where the indexer has not heard it. A waiting manifest becomes refused here only when the " +
"running indexer is the newest build of itself the trunk holds. With a digest it judges that one manifest of the " +
"module (a candidate's, or one an older judge refused) in the catalogue the trunk holds. indexer_verdicts reads the result.",
"running indexer is the newest build of itself the trunk holds. indexer_verdicts reads the result.",
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
"digest": map[string]any{"type": "string", "description": "one manifest digest of the module to judge (indexer_verdicts names it); none judges every manifest the trunk holds"},
"why": map[string]any{"type": "string", "description": "why it is judged again now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
key, value, err := index.NewJudgeAgain(str(args, "module"), str(args, "digest"), str(args, "why"), stdio.CallerOf(args), now())
key, value, err := index.NewJudgeAgain(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
if err != nil {
return nil, err
}
@@ -178,47 +150,8 @@ func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
}
}
// heldCounts reads the count of the old sections the indexer keeps.
func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) {
return func() (index.OldSectionsCount, error) {
e, err := s.Get(index.OldSectionsKey)
if err != nil {
return index.OldSectionsCount{}, fmt.Errorf("reading %s.%s: %w", index.OldSectionsBucket, index.OldSectionsKey, err)
}
if e == nil {
return index.OldSectionsCount{}, fmt.Errorf("the indexer has not counted the old sections yet: %s holds no %s", index.OldSectionsBucket, index.OldSectionsKey)
}
return index.DecodeOldSections(e.Value)
}
}
// keptRead is a state the tools bundle reads, as the index package reads a store: Get and Keys; it writes nothing.
type keptRead struct{ s stdio.KeptState }
func (k keptRead) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
e, err := k.s.Get(key)
if err != nil || e == nil {
return nil, 0, false, err
}
return e.Value, e.Revision, true, nil
}
func (k keptRead) Keys(context.Context) ([]string, error) { return k.s.Keys() }
func (keptRead) Create(context.Context, string, []byte) error { return errReadOnly }
func (keptRead) Update(context.Context, string, []byte, uint64) error { return errReadOnly }
func (keptRead) Delete(context.Context, string, uint64) error { return errReadOnly }
var errReadOnly = errors.New("the tools bundle only reads this state")
func main() {
ask := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second)
tools := append(Tools(ask, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...)
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, ask, time.Now)...)
tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, ask, time.Now)...)
if err := stdio.Serve("", tools); err != nil {
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState))); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1)
}
@@ -1,32 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// mergedTool is the forge bridge's tool that answers a merge's merged message again (issue 1585): arguments repository
// and merge, answer {message}, the message whole as the bridge writes it on FORGE. The indexer's manifest lists it in
// invokes, which is the grant to call it.
const mergedTool = delivery.ForgeBridgeModule + ".bridge_merged_message"
// messageVia asks the forge bridge for a merge's merged message, for a held merge the indexer does not hold in memory (a
// restart, a new build). An error is the bridge's own words.
func messageVia(call toolCaller) func(ctx context.Context, repository, merge string) ([]byte, error) {
return func(_ context.Context, repository, merge string) ([]byte, error) {
raw, err := call(mergedTool, map[string]string{"repository": repository, "merge": merge})
if err != nil {
return nil, err
}
var a struct {
Message json.RawMessage `json:"message"`
}
if err := json.Unmarshal(raw, &a); err != nil || len(a.Message) == 0 {
return nil, fmt.Errorf("the answer of %s is not {message}: %v", mergedTool, err)
}
return a.Message, nil
}
}
@@ -9,10 +9,7 @@ import (
"io"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -20,15 +17,9 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"mesh-delivery-indexer/internal/index"
)
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
@@ -51,89 +42,22 @@ accounts {
// bus with the stream FORGE and the indexer's durable <node>_<module> on it, the buckets the controller makes from the
// manifest's state (mesh-delivery-indexer_candidates, history 10) and mesh-controller_builds holding build records, and
// no mesh-delivery-gate_source bucket.
//
// And the take-in step (ADR 0633, issue 1585 I8a, I8b): the indexer's states pending, verdicts, graph, taken and packages
// and its stream take-in are there, mesh-build-go reported its toolchain, and the node-engine's composition answers the
// build of the indexer that runs. A merge changing the indexer's own tree writes only the indexer's entry, its manifest,
// its verdict and its key of the graph (and the packages it publishes); the rest is pending, said take-in.pending and
// named by indexer_pending with the build it waits for. A group's first member is gathered meanwhile. Started again as
// that build, the indexer takes the revision in and leaves the group gathered; the group's second member and its end
// then make one take-in, said take-in.taken under the group's key.
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
var running atomic.Pointer[string]
first := "fp-the-indexer-that-runs"
running.Store(&first)
// The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by
// repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it.
var bridgeMessages sync.Map
js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) {
// The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them.
builds.end(t, "novox/mesh-delivery", "a", "modules/a")
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
}, func(ctx context.Context, js jetstream.JetStream) {
// The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11).
for bucket, history := range map[string]uint8{index.PendingBucket: 1, delivery.VerdictsBucket: delivery.VerdictsHistory,
graph.Bucket: 1, delivery.TakenBucket: 1, delivery.PackagesBucket: 1} {
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil {
t.Fatal(err)
}
}
// The take-in events are the indexer's ordinary events (ADR 0633 §5), on the path manifest-accepted takes.
for name, subject := range map[string]string{"INDEXER_EVENTS": "mesh.mod.mesh-delivery-indexer.event.>"} {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: name, Subjects: []string{subject}}); err != nil {
t.Fatal(err)
}
}
tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.ToolchainKey("n1", "go")
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
if err != nil {
t.Fatal(err)
}
if _, err := tc.Put(ctx, key, value); err != nil {
t.Fatal(err)
}
// The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there.
answer := func(m *nats.Msg) {
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"})
_ = m.Respond(raw)
}
bridge := func(m *nats.Msg) {
var ask struct{ Repository, Merge string }
_ = json.Unmarshal(m.Data, &ask)
data, ok := bridgeMessages.Load(ask.Repository + "@" + ask.Merge)
if !ok {
raw, _ := json.Marshal(map[string]any{"error": "the gate holds no merge of " + ask.Repository + " with that commit"})
_ = m.Respond(raw)
return
}
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"message": json.RawMessage(data.([]byte))}, "node": "n1"})
_ = m.Respond(raw)
}
for _, subject := range []string{"mesh.mod.mesh-delivery-forge-bridge.call.bridge_merged_message.>", "mesh.mod.mesh-delivery-forge-bridge.tool.bridge_merged_message"} {
if _, err := js.Conn().Subscribe(subject, bridge); err != nil {
t.Fatal(err)
}
}
for _, subject := range []string{"mesh.seat.node-engine.call.composition.n1.>", "mesh.seat.node-engine.tool.composition.n1"} {
if _, err := js.Conn().Subscribe(subject, answer); err != nil {
t.Fatal(err)
}
}
})
h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef"
for _, f := range []forgemsg.Forge{
{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true},
{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true},
for _, f := range []delivery.Forge{
{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true},
{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true},
} {
data, err := forgemsg.EncodeForge(f)
data, err := delivery.EncodeForge(f)
if err != nil {
t.Fatal(err)
}
event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
event, _ := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data); err != nil {
t.Fatal(err)
}
@@ -159,202 +83,11 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
t.Errorf("%s#%d modules = %v, want [%s]", p.Repository, p.Number, p.Modules, want)
}
}
// I8a: pull request 12 merged, changing the indexer's own tree and a's.
manifest, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
const ix = delivery.IndexerModule
m1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1,
Merge: "1111111111111111111111111111111111111111", Root: "r0",
Trees: map[string]string{"modules/" + ix: "tree-indexer-2", "modules/a": "tree-a-2"},
Manifests: map[string]string{"modules/" + ix: string(manifest), "modules/a": `{"module":"a","version":"1"}`},
Locks: map[string]map[string]string{"modules/" + ix: {"go.sum": "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"}}}
merged := func(f forgemsg.Forge) uint64 {
t.Helper()
data, err := forgemsg.EncodeForge(f)
if err != nil {
t.Fatal(err)
}
event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgeMerged)
bridgeMessages.Store(f.Repository+"@"+f.Merge, data)
ack, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data)
if err != nil {
t.Fatal(err)
}
return ack.Sequence
}
m1Key := takein.PendingKey(merged(m1))
events, err := js.Stream(ctx, "INDEXER_EVENTS")
if err != nil {
t.Fatal(err)
}
// saidAll is every take-in event of kind said under key so far, oldest first.
saidAll := func(kind, key string) []takein.TakeIn {
event, _ := takein.TakeInEvent(kind)
info, err := events.Info(ctx)
if err != nil {
t.Fatal(err)
}
var out []takein.TakeIn
for seq := info.State.FirstSeq; seq <= info.State.LastSeq && seq > 0; seq++ {
msg, err := events.GetMsg(ctx, seq)
if err != nil || msg.Subject != "mesh.mod."+ix+".event."+event {
continue
}
if ti, err := takein.DecodeTakeIn(msg.Data); err == nil && ti.Key == key {
out = append(out, ti)
}
}
return out
}
// said is the last take-in event under kind and key, within 20 s.
said := func(kind, key string) (takein.TakeIn, bool) {
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if all := saidAll(kind, key); len(all) > 0 {
return all[len(all)-1], true
}
}
return takein.TakeIn{}, false
}
held, ok := said(takein.TakeInPending, m1Key)
if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first)
}
named := held.WaitsFor
get := func(bucket, key string) ([]byte, bool) {
t.Helper()
kv, err := js.KeyValue(ctx, bucket)
if err != nil {
t.Fatal(err)
}
e, err := kv.Get(ctx, key)
if err != nil {
return nil, false
}
return e.Value(), true
}
// Only what building and moving the named indexer needs is written.
raw, ok := get(delivery.ModulesBucket, ix)
own, err := delivery.DecodeModule(raw)
if !ok || err != nil || own.FP2 != named || own.Commit != m1.Merge {
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
}
if _, ok := get(delivery.ManifestsBucket, own.Manifest); !ok {
t.Errorf("manifests.%s is not written", own.Manifest)
}
if _, ok := get(delivery.VerdictsBucket, own.Manifest); !ok {
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
}
if gk, _ := graph.Key(ix); !func() bool { _, ok := get(graph.Bucket, gk); return ok }() {
t.Errorf("the indexer's key %s of the module graph is not written", gk)
}
repoKey, _ := delivery.RepositoryKey(m1.Repository)
if _, ok := get(delivery.ModulesBucket, "a"); ok {
t.Errorf("modules.a is written while the revision is pending")
}
if _, ok := get(delivery.TakenBucket, repoKey); ok {
t.Errorf("taken.%s is written while the revision is pending", repoKey)
}
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
} else if c, err := delivery.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
}
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
members := []delivery.Head{{Repository: "novox/mesh-catalog", Number: 40, Head: "4040404040404040404040404040404040404040"},
{Repository: "novox/mesh-controller", Number: 41, Head: "4141414141414141414141414141414141414141"}}
g1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/g", Number: 40, Head: members[0].Head,
Merge: "4000000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/c": "tree-c-2"},
Manifests: map[string]string{"modules/c": `{"module":"c","version":"1"}`}, Group: "feat/g", Members: members}
g2 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-controller", Branch: "feat/g", Number: 41, Head: members[1].Head,
Merge: "4100000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/k": "tree-k-1"},
Manifests: map[string]string{"modules/k": `{"module":"k","version":"1"}`}, Group: "feat/g", Members: members}
g1Key, groupKey := takein.PendingKey(merged(g1)), takein.GroupTakeInKey("feat/g")
if gathered, ok := said(takein.TakeInPending, groupKey); !ok || gathered.What != takein.TakeInGroup || len(gathered.Merges) != 1 {
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the group with its first member", groupKey, gathered, ok)
}
// indexer_pending, the tool over the same state, names both waits.
pendingKV, err := js.KeyValue(ctx, index.PendingBucket)
if err != nil {
t.Fatal(err)
}
waits := func() []takein.PendingWait {
t.Helper()
answer, err := index.PendingTools(kvStore{pendingKV}, nil, nil, time.Now)[0].Run(map[string]any{})
if err != nil {
t.Fatal(err)
}
return answer.(takein.PendingAnswer).Waits
}
var w []takein.PendingWait
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" {
break
}
}
if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) ||
w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent {
t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named)
}
// The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group.
running.Store(&named)
restart()
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if w = waits(); len(w) == 1 {
break
}
}
if raw, ok := get(delivery.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
}
if _, ok := get(delivery.TakenBucket, repoKey); !ok {
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
}
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
}
if _, ok := get(delivery.ModulesBucket, "c"); ok {
t.Errorf("modules.c is written while its group is gathered")
}
// I8b: the group's second member and its end: one take-in of both, said under the group's key.
merged(g2)
end, err := forgemsg.EncodeForgeGroupEnd(forgemsg.ForgeGroupEnd{Group: "feat/g", Members: []forgemsg.GroupMerge{
{Repository: g1.Repository, Number: g1.Number, Merge: g1.Merge}, {Repository: g2.Repository, Number: g2.Number, Merge: g2.Merge}}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+forgemsg.ForgeGroupEndEvent, end); err != nil {
t.Fatal(err)
}
if taken, ok := said(takein.TakeInTaken, groupKey); !ok || taken.What != takein.TakeInGroup || len(taken.Merges) != 2 {
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the group's end, want both members", groupKey, taken, ok)
}
var hasC, hasK bool
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) {
_, hasC = get(delivery.ModulesBucket, "c")
_, hasK = get(delivery.ModulesBucket, "k")
w = waits()
}
if !hasC || !hasK || len(w) != 0 {
t.Errorf("after the group's end: modules.c %v, modules.k %v, indexer_pending %+v: want both taken in and nothing waiting", hasC, hasK, w)
}
// One take-in: neither member was said as a revision of its own.
if len(saidAll(takein.TakeInPending, g1Key)) != 0 {
t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key)
}
}
// startIndexerRun starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
// mesh-controller_builds, and runs the program; restart starts it again. mesh-delivery-gate_source is never created.
func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue, func()) {
// startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
// mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created.
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
if err != nil {
t.Fatal(err)
@@ -433,24 +166,11 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before .
t.Fatal(err)
}
env := map[string]string{"MESH_BROKER_FILE": broker}
var stop func()
start := func() {
runCtx, cancel := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
stop = func() { cancel(); <-done }
}
start()
t.Cleanup(func() { stop() })
// restart stops the program and starts it again on the same bus, as the node-engine does with a new build.
restart := func() { stop(); start() }
runCtx, stop := context.WithCancel(ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
return js, ctx, candidates, modules, restart
}
// startIndexer is startIndexerRun for a test that never starts the program again.
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...)
return js, ctx, candidates, modules
}
@@ -482,7 +202,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent")
})
const merge = "89abcdef0123456789abcdef0123456789abcdef"
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"},
Manifests: map[string]string{"modules/a": `{"module":"a","version":"1"}`}})
if err != nil {
@@ -509,7 +229,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
// its inputs and the reason it has no fingerprint of the new form, and manifests.<digest> with the manifest byte for byte.
//
// And it judges them (novox/hq ADR 0578, issue 1585): with three manifests more, one valid, one refused and one with a
// field the decoder does not know (it waits, ADR 0601 §3), mesh-sdk's bus reader read before the indexer judged composes all three, the field set
// field the decoder does not know, mesh-sdk's bus reader read before the indexer judged composes all three, the field set
// aside and named (a missing verdict is accepted, §3); the indexer writes the three verdicts, says manifest-refused for
// each refusal, and the reader then composes the valid one and holds the others, named, never ending its read.
func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 1585
@@ -536,7 +256,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
t.Fatal(err)
}
})
forge := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
forge := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
Trees: map[string]string{"modules/avahi": "newtree"},
Manifests: map[string]string{"modules/avahi": string(manifest)},
@@ -544,7 +264,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
for name, raw := range judged {
forge.Manifests["modules/"+name], forge.Trees["modules/"+name] = raw, "tree-"+name
}
data, err := forgemsg.EncodeForge(forge)
data, err := delivery.EncodeForge(forge)
if err != nil {
t.Fatal(err)
}
@@ -580,7 +300,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
if err != nil {
t.Fatal(err)
}
want := map[string]string{"good": delivery.VerdictAccepted, "bad": delivery.VerdictRefused, "later": delivery.VerdictManifestWaits}
want := map[string]string{"good": delivery.VerdictAccepted, "bad": delivery.VerdictRefused, "later": delivery.VerdictRefused}
got := map[string]delivery.ManifestVerdict{}
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && len(got) < len(want); time.Sleep(50 * time.Millisecond) {
for name := range want {
@@ -596,12 +316,10 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
t.Errorf("the verdict on %s = %+v within 10 s, want %s by the running judge", name, v, state)
}
}
// ADR 0601 §3: the field this judge does not know waits and is named, since no node-engine answers which build of the
// indexer runs here (the unit tests of the index package prove it refused once that build is the trunk's).
if !strings.Contains(strings.Join(got["bad"].Problems, " "), `provides "amqp"`) || strings.Join(got["later"].Unknown, " ") != "a-field-from-later" {
t.Errorf("the refusal names no problem or the wait no field: %+v", got)
if !strings.Contains(strings.Join(got["bad"].Problems, " "), `provides "amqp"`) || !strings.Contains(strings.Join(got["later"].Problems, " "), "a-field-from-later") {
t.Errorf("the refusals name no problem: %+v", got)
}
// manifest-refused, said once for each refusal (a wait is no refusal).
// manifest-refused, said once for each refusal.
events, err := js.Stream(ctx, "INDEXER_EVENTS")
if err != nil {
t.Fatal(err)
@@ -617,8 +335,8 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
said[r.Module] = true
}
}
if !said["bad"] || said["later"] || said["good"] {
t.Errorf("manifest-refused said for %v, want bad only", said)
if !said["bad"] || !said["later"] || said["good"] {
t.Errorf("manifest-refused said for %v, want bad and later", said)
}
// How a composer holds a refused module by these verdicts is the node-engine's to test (ADR 0593 §7: the composer
// and its bus reader are its own).
@@ -638,7 +356,7 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) {
builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi")
})
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
Trees: map[string]string{"modules/avahi": "newtree"},
Manifests: map[string]string{"modules/avahi": string(manifest)},
@@ -725,7 +443,7 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
t.Fatal(err)
}
})
pull, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: forgemsg.PullUpdated,
pull, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: delivery.PullUpdated,
Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": string(manifest)}})
if err != nil {
t.Fatal(err)
@@ -895,7 +613,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
place(ctx, t, "n1", "mesh-build-go")
place(ctx, t, "n2")
})
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
Trees: map[string]string{"modules/avahi": "newtree"},
Manifests: map[string]string{"modules/avahi": string(manifest)},
@@ -24,11 +24,8 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/sdk/graph"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/internal/index"
@@ -154,34 +151,6 @@ type delivered interface {
Metadata() (*jetstream.MsgMetadata, error)
}
// groupEndSubject is the whole subject of a delivery group's end on the forge bridge's stream: compared exactly, so a
// repository named forge/group, whose merged message ends the same way, is never taken for one.
var groupEndSubject = process.ModuleStreamSubject(delivery.ForgeBridgeModule, forgemsg.ForgeGroupEndEvent)
// handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end
// whose members are not all held yet, or while the build this program runs is not known, is left for redelivery.
func handleGroupEnd(ctx context.Context, m delivered, ix *index.Indexer, h *index.Heard, logger *log.Logger) {
end, err := forgemsg.DecodeForgeGroupEnd(m.Data())
if err != nil {
logger.Printf("%s: not a group's end, dropped: %v", m.Subject(), err)
h.Outcome, h.Words = index.HeardDropped, err.Error()
_ = m.Term()
return
}
h.Kind, h.Branch = forgemsg.ForgeGroupEndEvent, end.Group
applied, words, err := ix.EndGroup(ctx, end)
h.Written, h.Late = applied.Written, applied.Late
if err != nil {
logger.Printf("%s: the end of the group %q left for redelivery: %v", m.Subject(), end.Group, err)
h.Outcome, h.Words = index.HeardLeft, err.Error()
return
}
h.Outcome, h.Words = index.HeardApplied, words
if err := m.DoubleAck(ctx); err != nil {
logger.Printf("%s: the end of the group %q applied, its acknowledgement was not confirmed: %v", m.Subject(), end.Group, err)
}
}
// handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated,
// never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server
// delivers it again (the server's ack wait is the only clock). Each message handled is written to the heard state
@@ -200,12 +169,7 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto
logger.Printf("%s: sequence %d is not written to the heard state: %v", h.Subject, h.Sequence, err)
}
}()
// A delivery group's end (novox/hq ADR 0633 §4) has a body of its own: told apart by its name, before the decode.
if m.Subject() == groupEndSubject {
handleGroupEnd(ctx, m, ix, &h, logger)
return
}
f, err := forgemsg.DecodeForge(m.Data())
f, err := delivery.DecodeForge(m.Data())
if err != nil {
logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err)
h.Outcome, h.Words = index.HeardDropped, err.Error()
@@ -216,13 +180,7 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto
if f.Merge != "" {
h.Commit = f.Merge
}
var applied index.Applied
if f.Kind == forgemsg.ForgeMerged && h.Sequence > 0 {
// A merge is taken in only by the indexer its revision names (ADR 0633 §1): kept in pending until then.
applied, h.Words, err = ix.Arrive(ctx, h.Sequence, f)
} else {
applied, err = ix.ApplyMessage(ctx, f)
}
applied, err := ix.ApplyMessage(ctx, f)
h.Written, h.Late = applied.Written, applied.Late
switch {
case errors.Is(err, index.ErrUnreadable):
@@ -338,27 +296,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
// fingerprint takes the version line.
packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
// How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then
// the judge refuses no use of an old section.
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
// The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written.
graph := process.AwaitBucket(ctx, js, conn, graph.Bucket, "no module graph is written", logger.Printf)
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf)
ix := &index.Indexer{OwnBuildState: awaitedStore{ownBuildState}, TryRecovery: awaitedStore{tryRecovery}, OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf),
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), ReadFile: readFileVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), Graph: awaitedStore{graph}, Message: messageVia(calls.Call),
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before.
Pending: awaitedStore{process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
SayTakeIn: sayTakeIn(js, cred)}
// Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
// revisions in with its old judge.
ix.FixOwnBuild(ctx)
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node)}
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
var started atomic.Bool
@@ -418,9 +361,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if !gate.Toolchain(module, key, value, deleted) {
return nil
}
// The modules written again, and pending evaluated again (ADR 0633 §1): the build a revision names moves with
// the toolchain.
return ix.ToolchainReported(ctx)
return ix.Recompute(ctx)
},
recompute: ix.Recompute}, logger)
if err := tools.start(); err != nil {
@@ -500,27 +441,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer tc.Stop()
}
// The consumer on the checker's stream exists before the open tries are listed and is heard after (ADR 0638 §3): an event
// said between the call and the hearing waits in it, and a try that started or ended meanwhile is found either way.
pcons, perr := cred.EventConsumer(ctx, js, PreviewsStream)
if perr != nil {
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews and tries are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, perr)
}
// The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's
// stream starts (ADR 0633 §2). When the build this program runs is not known yet, nothing waits for it: the consumer
// starts, and a merged message left unacknowledged asks the build again at its redelivery (clause 1). A hand take-in
// (indexer_take_in_pending) is answered at any time: the takeins watch runs already.
if _, words, err := ix.TakeInAtStart(ctx); err != nil {
return fmt.Errorf("the take-in at start: %w", err)
} else if words != "" {
logger.Printf("take-in at start: %s", words)
}
// The open tries are found again through the checker's tool: a try event missed while the indexer was down would leave a
// try with no candidate, or a candidate with no try. Not fatal: the events come, and a preview written writes the candidates.
// A recovery that fails is said (journal, and indexer_try_candidates) and the events are heard all the same.
_ = ix.RecoverTries(ctx)
if perr == nil {
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf)
switch pcons, err := cred.EventConsumer(ctx, js, PreviewsStream); {
case err != nil:
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err)
default:
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.Written(m) }, logger.Printf)
if err != nil {
return err
}
@@ -11,7 +11,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
@@ -74,8 +73,8 @@ const head = "0123456789abcdef0123456789abcdef01234567"
func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
st := &memStore{vals: map[string][]byte{}}
ix := &index.Indexer{Store: st, Builds: index.NewBuilds()}
data, _ := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
data, _ := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
heard := &memStore{vals: map[string][]byte{}}
handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0))
@@ -85,7 +84,7 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
}
// Issue 1126 note 76: what the indexer heard of FORGE is readable (indexer_heard): subject, sequence, deliveries, time,
// repository, commit and what it did.
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != forgemsg.ForgePull ||
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != delivery.ForgePull ||
h.Repository != "novox/mesh-delivery" || h.Number != 12 || h.Commit != head || h.Outcome != index.HeardApplied || h.Handled.IsZero() {
t.Errorf("heard = %+v", h)
}
@@ -103,31 +102,6 @@ func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 12
if h := heardOf(t, heard); h.Outcome != index.HeardDropped || h.Sequence != 41 || h.Words == "" {
t.Errorf("heard = %+v", h)
}
// 1585/T9 (ADR 0633 §4): a group's end is told apart by its name before the forge decode: one that reads is acked
// (an indexer with no pending state takes the members in one by one), one that does not is dropped as a group's end.
for _, c := range []struct {
data string
termd int
logged string
}{{`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`, 0, ""},
{`{"v":1,"group":"feat/x"}`, 1, "not a group's end"}} {
logs.Reset()
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event." + forgemsg.ForgeGroupEndEvent, data: []byte(c.data)}
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
if m.termd != c.termd || m.acked+m.termd != 1 || !strings.Contains(logs.String(), c.logged) {
t.Errorf("group end %s: terminated %d acked %d log %q", c.data, m.termd, m.acked, logs.String())
}
if h := heardOf(t, heard); h.Kind != forgemsg.ForgeGroupEndEvent && c.termd == 0 {
t.Errorf("heard of a group's end = %+v", h)
}
}
// The repository forge/group's merged message ends as a group's end does: it is a forge message all the same.
logs.Reset()
m = &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.forge.group.merged", data: []byte(`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`)}
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
if !strings.Contains(logs.String(), "not a forge message") {
t.Errorf("forge/group's merged message read as a group's end: %q", logs.String())
}
}
func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value)
@@ -9,8 +9,6 @@ import (
"path"
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
@@ -20,7 +18,7 @@ import (
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
// (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each
// path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk
// forgemsg.Forge.Paths; novox/hq ADR 0499 clause 3).
// delivery.Forge.Paths; novox/hq ADR 0499 clause 3).
const commitTreesTool = "gitea.gitea_commit_trees"
// toolCaller calls a module's tool, `<module>.<tool>`, as the indexer's own account (mesh-sdk process.Conn.Call).
@@ -102,7 +100,7 @@ func filesVia(call toolCaller) func(ctx context.Context, repository, sha string,
}
for _, d := range dirs {
if _, ok := a.Files[d]; !ok {
return nil, fmt.Errorf("the answer of %s lists no files of %q: %w", commitTreesTool, d, index.ErrNotInTree)
return nil, fmt.Errorf("the answer of %s lists no files of %q", commitTreesTool, d)
}
}
return a.Files, nil
@@ -121,12 +119,6 @@ func goModVia(call toolCaller) func(ctx context.Context, repository, sha, dir st
return goFileVia(call, "go.mod")
}
// readFileVia reads one file of a repository at a commit by its path (gitea_get_file); a file the commit does not hold is
// none. The seat definitions of a revision are read with it (seatsof.go).
func readFileVia(call toolCaller) func(ctx context.Context, repository, sha, path string) ([]byte, error) {
return goFileVia(call, "")
}
// goFileVia reads one Go file of a module directory at a commit through the forge's get-file tool; a file the commit does
// not hold is none.
func goFileVia(call toolCaller, name string) func(ctx context.Context, repository, sha, dir string) ([]byte, error) {
@@ -136,10 +128,7 @@ func goFileVia(call toolCaller, name string) func(ctx context.Context, repositor
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
p := name
switch {
case name == "": // readFileVia: dir is the file's whole path
p = dir
case dir != "" && dir != delivery.RootPath:
if dir != "" && dir != delivery.RootPath {
p = dir + "/" + name
}
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
@@ -269,8 +258,8 @@ const (
)
// openVia asks the forge which pull requests into the trunk are open now (gitea_open_pulls), every repository's.
func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, error) {
return func(context.Context) ([]forgemsg.OpenPull, error) {
func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, error) {
return func(context.Context) ([]delivery.OpenPull, error) {
raw, err := call(openPullsTool, map[string]any{})
if err != nil {
return nil, err
@@ -286,40 +275,12 @@ func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, er
if err := json.Unmarshal(raw, &o); err != nil || o.Pulls == nil {
return nil, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
}
out := []forgemsg.OpenPull{}
out := []delivery.OpenPull{}
for _, p := range *o.Pulls {
if p.Base == trunkBranch && p.Number > 0 {
out = append(out, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
out = append(out, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
}
}
return out, nil
}
}
// openTriesTool is the checker's tool that lists the tries that wait or run (mesh-delivery-checker checker_tries).
const openTriesTool = "mesh-delivery-checker.checker_tries"
// openTriesVia lists the open tries through the checker's tool, never its state (ADR 0596).
func openTriesVia(call toolCaller) func(ctx context.Context) ([]tryevent.Open, error) {
return func(context.Context) ([]tryevent.Open, error) {
raw, err := call(openTriesTool, map[string]any{})
if err != nil {
return nil, err
}
// No try open is an answer: the checker's list is empty, or null in an older checker (issue 1921). An answer without the
// key is none.
var o map[string]json.RawMessage
if err := json.Unmarshal(raw, &o); err != nil {
return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err)
}
list, ok := o["tries"]
if !ok {
return nil, fmt.Errorf("the answer of %s is not {tries}: no tries key", openTriesTool)
}
var tries []tryevent.Open
if err := json.Unmarshal(list, &tries); err != nil {
return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err)
}
return tries, nil
}
}
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
"mesh-delivery-indexer/internal/index"
"slices"
"strings"
"testing"
@@ -51,15 +50,6 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
if _, has := gotArgs.(map[string]any)["paths"]; has {
t.Error("the trunk asked for paths")
}
// A directory the answer lists no files of is a failure of the tree (index.ErrNotInTree); a call that fails is not.
noDir := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"files":{}}`), nil }
if _, err := filesVia(noDir)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); !errors.Is(err, index.ErrNotInTree) {
t.Errorf("a missing directory: %v, want ErrNotInTree", err)
}
down := func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }
if _, err := filesVia(down)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); err == nil || errors.Is(err, index.ErrNotInTree) {
t.Errorf("a failed call: %v, want an error that is not ErrNotInTree", err)
}
// The build the indexer runs is the engine's word (Runs.FP of its composition verb), asked of its own machine.
var asked string
own := func(key string, body any) (json.RawMessage, error) {
@@ -69,45 +59,10 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
if fp, err := ownBuildVia(own, "n1")(context.Background()); err != nil || fp != "src2:abc" || asked != "seat:node-engine.composition@n1" {
t.Errorf("own build = %q %v asked %q", fp, err, asked)
}
// Issue 1921: the start during a replacement. What runs still names the old build, the want (composed by the node itself)
// the successor, which this process is: it fixes the successor. A want that is not self, or is removed, leaves what runs.
for _, row := range []struct{ name, answer, want string }{
{"replacing: runs old, want new and self", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true}}`, "src2:new"},
{"a want the node does not compose", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new"}}`, "src2:old"},
{"replacing: the apply of the want failed or is held", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:new","held":true}}`, "src2:old"},
{"replacing: an attempt of another build", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:older"}}`, "src2:new"},
{"the want already runs", `{"runs":{"fp":"src2:new"},"want":{"fp":"src2:new","self":true}}`, "src2:new"},
{"a removed want", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true,"removed":true}}`, "src2:old"},
{"no build runs yet, want self", `{"want":{"fp":"src2:new","self":true}}`, "src2:new"},
} {
ask := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil }
if fp, err := ownBuildVia(ask, "n1")(context.Background()); err != nil || fp != row.want {
t.Errorf("%s: own build = %q %v, want %q", row.name, fp, err, row.want)
}
}
none := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }
if _, err := ownBuildVia(none, "n1")(context.Background()); err == nil {
t.Error("an engine that says nothing runs gave a build")
}
// Issue 1921: the checker's list of open tries; none open is an answer, whether [] or null.
for _, row := range []struct {
name, answer string
want int
refused bool
}{
{"one open", `{"tries":[{"id":"try-1","state":"running","whole":true,"members":[{"repository":"novox/a","head":"h","trunk":"t"}]}]}`, 1, false},
{"an empty list", `{"tries":[]}`, 0, false},
{"null, as an older checker answers none", `{"tries":null}`, 0, false},
{"no tries key", `{}`, 0, true},
{"not json", `nope`, 0, true},
{"tries of another shape", `{"tries":{"a":1}}`, 0, true},
} {
call := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil }
got, err := openTriesVia(call)(context.Background())
if (err != nil) != row.refused || len(got) != row.want {
t.Errorf("%s: %d tries, error %v; want %d, refused %v", row.name, len(got), err, row.want, row.refused)
}
}
}
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
@@ -25,32 +25,10 @@ func ownBuildVia(call toolCaller, node string) func(ctx context.Context) (string
Runs *struct {
FP string `json:"fp"`
} `json:"runs"`
Want *struct {
FP string `json:"fp"`
Self bool `json:"self"`
Removed bool `json:"removed"`
} `json:"want"`
// Attempt is the engine's last apply that did not end running (failed, or held), with the build it was of.
Attempt *struct {
FP string `json:"fp"`
} `json:"attempt"`
}
if err := json.Unmarshal(raw, &a); err != nil {
return "", fmt.Errorf("the answer of %s is not a composition: %w", compositionVerb, err)
}
// While the engine's apply of the want is in progress (what runs still names the old build, the want names another, and no
// attempt names that want as failed or held), this process is the successor and fixes the want's build. self is the node's
// standing way of composing the module, not a mark of this process, so it is not enough alone: an old process started again
// while the want names a successor that does not run (its apply failed or is held) takes what runs. The start fixes the
// build once (quick fix, ADR 0639 §7: nothing is learnt later, nothing waits on a timer); the thorough form is the
// engine passing each process the fingerprint it starts it with.
running := ""
if a.Runs != nil {
running = a.Runs.FP
}
if w := a.Want; w != nil && w.Self && !w.Removed && w.FP != "" && w.FP != running && (a.Attempt == nil || a.Attempt.FP != w.FP) {
return w.FP, nil
}
if a.Runs == nil || a.Runs.FP == "" {
return "", errors.New("the engine says no fingerprint of the indexer runs")
}
@@ -13,7 +13,6 @@ import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -72,10 +71,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
}
serveForge(t, js.Conn(), listing, goSumContent)
})
merged := func(repo string, number int, merge string, f forgemsg.Forge) {
merged := func(repo string, number int, merge string, f delivery.Forge) {
t.Helper()
f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = forgemsg.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge
data, err := forgemsg.EncodeForge(f)
f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = delivery.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge
data, err := delivery.EncodeForge(f)
if err != nil {
t.Fatal(err)
}
@@ -102,10 +101,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
"tsconfig.json": "5555555555555555555555555555555555555555"}}
}
// mesh-sdk at m1, then the catalogue's two modules: their go.sum names mesh-sdk, read by the indexer itself.
merged("novox/mesh-sdk", 70, m1, forgemsg.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1},
merged("novox/mesh-sdk", 70, m1, delivery.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1},
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("6666666666666666666666666666666666666666")})
module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m1 })
merged("novox/mesh-catalog", 301, catalogMerge, forgemsg.Forge{Root: "r0",
merged("novox/mesh-catalog", 301, catalogMerge, delivery.Forge{Root: "r0",
Trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"},
Manifests: map[string]string{"modules/importer": named("importer"), "modules/plain": named("plain")},
Locks: map[string]map[string]string{"modules/importer": {"go.sum": goSumHash}, "modules/plain": {"go.sum": goSumHash}}})
@@ -141,11 +140,11 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
// over its named tree, is not written (issue 1758, ADR 0499 clause 6; the operator, 2026-10-25: a module's value moves
// only with the module).
const goOnly = "4444444444444444444444444444444444444444"
merged("novox/mesh-sdk", 72, goOnly, forgemsg.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc",
merged("novox/mesh-sdk", 72, goOnly, delivery.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc",
Trees: map[string]string{"": "cccccccccccccccccccccccccccccccccccccccc"}, Manifests: map[string]string{"": string(sdkManifest)},
Paths: sdkObjects("6666666666666666666666666666666666666666")})
// mesh-sdk merges a change of buildstep alone (its version moves: src changed too).
merged("novox/mesh-sdk", 71, m2, forgemsg.Forge{Root: sdk2, Trees: map[string]string{"": sdk2},
merged("novox/mesh-sdk", 71, m2, delivery.Forge{Root: sdk2, Trees: map[string]string{"": sdk2},
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("7777777777777777777777777777777777777777")})
sdkAt2 := module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m2 })
if sdkAt2.Inputs2 == nil || !strings.HasPrefix(sdkAt2.Inputs2.Tree, delivery.NamedTreePrefix) {
@@ -11,7 +11,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
@@ -74,10 +73,7 @@ func (w *previewWatch) ensure() error {
}
})
}()
if err := w.ix.RecomputeCandidates(w.ctx); err != nil {
return err
}
return w.ix.WriteTryCandidates(w.ctx)
return w.ix.RecomputeCandidates(w.ctx)
}
// Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the
@@ -94,44 +90,3 @@ func (w *previewWatch) Written(m delivered) {
}
_ = m.Ack()
}
// tryEventSubject is the subject of an event of the checker's stream checks about a try.
func tryEventSubject(event string) string {
return "mesh.stream." + delivery.CheckerModule + ".event." + event
}
// heardChecks takes one event of the checker's stream in: a preview written (Written), or a whole try's start or end, from
// which the candidate of its group is written or deleted (index.Indexer.StartTry, EndTry). A failure stays unacknowledged, and
// the server delivers it again; a body that is no try event is terminated.
func (w *previewWatch) heardChecks(m delivered) {
var event string
switch m.Subject() {
case tryEventSubject(tryevent.Started):
event = tryevent.Started
case tryEventSubject(tryevent.Ended):
event = tryevent.Ended
default:
w.Written(m)
return
}
e, err := tryevent.Decode(event, m.Data())
if err != nil {
w.logger.Printf("%s: not a try event, dropped: %v", m.Subject(), err)
_ = m.Term()
return
}
if event == tryevent.Started {
var ms []index.TryMember
for _, x := range e.Members {
ms = append(ms, index.TryMember{Repository: x.Repository, Head: x.Head, Trunk: x.Trunk})
}
err = w.ix.StartTry(w.ctx, e.ID, ms)
} else {
err = w.ix.EndTry(w.ctx, e.ID)
}
if err != nil {
w.logger.Printf("%s of %s: left for redelivery: %v", event, e.ID, err)
return
}
_ = m.Ack()
}
@@ -14,13 +14,7 @@ import (
// takerIn is the part of the indexer a takeins value asks through.
type takerIn interface {
TakeIn(ctx context.Context, module string) index.TakenIn
TakeInRepository(ctx context.Context, repository string) index.TakenIn
JudgeAgain(ctx context.Context, module, digest string) index.TakenIn
TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn
DropPending(ctx context.Context, key, by, why string) index.TakenIn
TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn
DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn
RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn
JudgeAgain(ctx context.Context, module string) index.TakenIn
}
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
@@ -39,26 +33,8 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
return
}
asked, r := "take-in", index.TakenIn{}
if a.RecoverTries {
asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why)
a.Module = "tries"
} else if a.TryCandidate != "" && a.Drop {
asked, r = "drop of a try's candidate", ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why)
a.Module = a.TryCandidate
} else if a.Merge != "" {
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
a.Module = r.Module
} else if a.Pending != "" && a.Drop {
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why)
a.Module = a.Pending
} else if a.Pending != "" {
asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why)
a.Module = a.Pending
} else if a.Repository != "" {
asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository)
a.Module = a.Repository
} else if a.Judge {
asked, r = "judging again", ix.JudgeAgain(ctx, a.Module, a.Digest)
if a.Judge {
asked, r = "judging again", ix.JudgeAgain(ctx, a.Module)
} else {
r = ix.TakeIn(ctx, a.Module)
}
@@ -6,7 +6,6 @@ import (
"fmt"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"github.com/nats-io/nats.go"
@@ -37,54 +36,3 @@ func sayRefused(js jetstream.JetStream, cred process.Account, logf func(string,
}
}
}
// sayAccepted says the indexer's event manifest-accepted (issue 1966 D3, novox/hq ADR 0595 §4):
// mesh.mod.<module>.event.manifest-accepted, carrying the module's bus-state entries whole, with the digest and judge as
// the bus's dedup id (delivery.ManifestAccepted.EventID), so one acceptance is said once.
func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, delivery.ManifestAccepted) {
return func(ctx context.Context, a delivery.ManifestAccepted) {
data, err := json.Marshal(a)
if err != nil {
logf("judge: %s for %s is not said: %v", delivery.ManifestAcceptedEvent, a.Module, err)
return
}
msg := nats.NewMsg(cred.EventSubject(delivery.ManifestAcceptedEvent))
id := a.EventID()
msg.Header.Set("x-event-id", id)
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
logf("judge: %s", fmt.Errorf("saying %s for %s: %w", delivery.ManifestAcceptedEvent, a.Module, err))
}
}
}
// sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events
// take: mesh.mod.<module>.event.take-in.<kind>, the wait's key in the body. A publish that fails is the caller's to say.
func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error {
return func(ctx context.Context, t takein.TakeIn) error {
event, err := takein.TakeInEvent(t.Kind)
if err != nil {
return err
}
data, err := takein.EncodeTakeIn(t)
if err != nil {
return err
}
msg := nats.NewMsg(cred.EventSubject(event))
msg.Header.Set("x-source", cred.Module)
msg.Header.Set("x-node", cred.Node)
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
msg.Header.Set("content-type", "application/json")
msg.Data = data
pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
_, err = js.PublishMsg(pub, msg)
return err
}
}
@@ -46,19 +46,7 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
fmt.Fprintln(stderr, "mesh-judge:", err)
return 1
}
// The request's seat definitions (ADR 0636 §4): the composer's request type of this build does not decode them.
var seats struct {
Seats []json.RawMessage `json:"seats"`
}
if err := json.Unmarshal(raw, &seats); err != nil {
fmt.Fprintln(stderr, "mesh-judge:", err)
return 1
}
files := make([][]byte, len(seats.Seats))
for i, s := range seats.Seats {
files[i] = s
}
v, err := judge.RequestWith(r, files)
v, err := judge.Request(r)
if err != nil {
fmt.Fprintln(stderr, "mesh-judge:", err)
return 1
+1 -13
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103120252-42bd7b802ac2
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
@@ -23,15 +23,3 @@ require (
)
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun
require (
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein v0.0.0-00010101000000-000000000000
git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
)
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein => ./takein
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ../mesh-delivery-forge-bridge/forgemsg

Some files were not shown because too many files have changed in this diff Show More