Compare commits
151
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1094a9265 | ||
|
|
4907470e90 | ||
|
|
55fb32ba25 | ||
|
|
cd01f1c196 | ||
|
|
261f8d54f7 | ||
|
|
643ec7cd91 | ||
|
|
c0e5bc2834 | ||
|
|
6f048a239c | ||
|
|
fab533dfd0 | ||
|
|
39285c95bd | ||
|
|
1b262125ee | ||
|
|
bfe4caf196 | ||
|
|
3742a7c54c | ||
|
|
f1e6520037 | ||
|
|
4ddf78395d | ||
|
|
8c67adaf91 | ||
|
|
e3735da928 | ||
|
|
78d57e1cc6 | ||
|
|
9f9e743b2f | ||
|
|
abe9ea3796 | ||
|
|
f9578cb7b4 | ||
|
|
69a2790099 | ||
|
|
88a47be251 | ||
|
|
bd5eb33e9e | ||
|
|
c2b8ac3370 | ||
|
|
92c38981ea | ||
|
|
45386212da | ||
|
|
b97e277d2d | ||
|
|
8823cc29d6 | ||
|
|
55936c1cd7 | ||
|
|
31ff708f88 | ||
|
|
2fcd61baa6 | ||
|
|
655791b8a3 | ||
|
|
3c832d1a2d | ||
|
|
0605962960 | ||
|
|
343321f11e | ||
|
|
4b7a31b2c7 | ||
|
|
4596fbe1b7 | ||
|
|
c78941e80b | ||
|
|
7b7096f37e | ||
|
|
4e4601a1eb | ||
|
|
a395b49be2 | ||
|
|
1040c8ec46 | ||
|
|
92dc5f03d0 | ||
|
|
ef13e63268 | ||
|
|
9c6e70e555 | ||
|
|
ad9a353484 | ||
|
|
a762701e3d | ||
|
|
d6b6f24e98 | ||
|
|
ed4f07b0d8 | ||
|
|
7609f2d9a2 | ||
|
|
d11a5af113 | ||
|
|
531d05ab4b | ||
|
|
071a28f218 | ||
|
|
72633722c5 | ||
|
|
0579e4e74d | ||
|
|
2cb3198049 | ||
|
|
ae31b66a9e | ||
|
|
fc300216ec | ||
|
|
c4bba04e8f | ||
|
|
1c6ac37fc1 | ||
|
|
dd90a26251 | ||
|
|
70ade15c36 | ||
|
|
cbbc10871f | ||
|
|
084bfb9f3e | ||
|
|
9d0b3cafc5 | ||
|
|
62f7375522 | ||
|
|
2d49c660c0 | ||
|
|
9d2816d001 | ||
|
|
7006380599 | ||
|
|
46c735473f | ||
|
|
702b19fdb1 | ||
|
|
0279c5c288 | ||
|
|
1d9f9cfa2c | ||
|
|
c45bbfc391 | ||
|
|
b4bfc14556 | ||
|
|
df05b1a5f7 | ||
|
|
509d277b32 | ||
|
|
477b3bfea6 | ||
|
|
ffdeb9771a | ||
|
|
b83085adac | ||
|
|
390e50d7d1 | ||
|
|
007c982ae6 | ||
|
|
e3ce4b095c | ||
|
|
4154c3040c | ||
|
|
945906ac97 | ||
|
|
1c66d58bce | ||
|
|
34ef185ba5 | ||
|
|
f12823681c | ||
|
|
b96a4f1d99 | ||
|
|
70ac4ff79a | ||
|
|
51deed51e3 | ||
|
|
3842a464ed | ||
|
|
877b09e92b | ||
|
|
453cdfa607 | ||
|
|
8f7945a109 | ||
|
|
2773885600 | ||
|
|
45bd63427f | ||
|
|
effa12c4b4 | ||
|
|
9da14440e8 | ||
|
|
fd638b99d5 | ||
|
|
0d1edef846 | ||
|
|
5c4d770aa1 | ||
|
|
b4855f21c3 | ||
|
|
1387b10e78 | ||
|
|
a1814c6571 | ||
|
|
92b4307922 | ||
|
|
2c37f48f9d | ||
|
|
4a736858b6 | ||
|
|
3411b91571 | ||
|
|
0ce16ad773 | ||
|
|
875d42fe3e | ||
|
|
aacdba44e3 | ||
|
|
39b3ce3c73 | ||
|
|
e1d0c9fe03 | ||
|
|
6bc16d74db | ||
|
|
9f49bc92ad | ||
|
|
4f6e1021c6 | ||
|
|
027c445bae | ||
|
|
7ba2668aa8 | ||
|
|
a901b840a0 | ||
|
|
f70faed5b6 | ||
|
|
5ee3b4c15f | ||
|
|
2f387a3f4c | ||
|
|
22aedbe97f | ||
|
|
478f0110e9 | ||
|
|
6d24922172 | ||
|
|
0fa00d5e6c | ||
|
|
a41d2851c7 | ||
|
|
ea5be10532 | ||
|
|
6b9d211f72 | ||
|
|
b342c67b19 | ||
|
|
daf178c4ca | ||
|
|
a9aa6d29ea | ||
|
|
ecf96be8ce | ||
|
|
ab3cb1ca9e | ||
|
|
054f089614 | ||
|
|
6301e49c73 | ||
|
|
78fe1ce960 | ||
|
|
6645ac58c2 | ||
|
|
c4470f42db | ||
|
|
4bb790bedf | ||
|
|
aa98729457 | ||
|
|
2fdb0516cd | ||
|
|
28a6858863 | ||
|
|
1ab0274547 | ||
|
|
07af86b153 | ||
|
|
dd2e7c8e64 | ||
|
|
756a3813e2 | ||
|
|
9a508701ff | ||
|
|
7ee7248fad |
@@ -27,6 +27,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/health"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
"mesh-delivery-assigner/internal/judge"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
@@ -434,6 +435,61 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
if !self {
|
||||
t.Error("ace's own want does not say it composes its own once placed")
|
||||
}
|
||||
|
||||
// ADR 0603 (L3, D15): one source revision changes a provider (db), its new consumer (app) and a build module (tool, whose
|
||||
// recipe moved: ace runs its first build and the second, under the agreed toolchain, is the one the source revision names).
|
||||
// The keys are written as their writers write them: the indexer's modules and manifests, the builder's builds, the
|
||||
// controller's grants, the placement seat's placement and the provisioner's provisions. The build module's want moves at
|
||||
// once and the provider's with it; the consumer's waits for its provision, with the words naming it, and moves once the
|
||||
// provisioner provides it (rule 8, ADR 0570 §4): nothing waits for a turn or an order, only for a key.
|
||||
offer := `"provides":[{"name":"database","scope":"mesh","answered-by":"helper"}]`
|
||||
dbDigest := b.manifest(t, "db", `{"module":"db","version":"1",`+offer+`}`)
|
||||
appDigest := b.manifest(t, "app", `{"module":"app","version":"1","requires":["database"]}`)
|
||||
toolDigest := b.manifest(t, "tool", `{"module":"tool","version":"1"}`)
|
||||
asked := func(state, why string) {
|
||||
raw, err := provisions.Encode(provisions.Provision{V: provisions.Version, Provision: "database", Consumer: provisions.End{Module: "app", Node: "ace"},
|
||||
Provider: provisions.End{Module: "db", Node: "ace"}, State: state, Why: why, At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.putJSON(t, provisions.ProvisionsBucket, "ace.app.database", raw)
|
||||
}
|
||||
// The provisioner keys it from the manifests, as soon as they are known and whether the provider is placed yet or not.
|
||||
asked(provisions.Asked, "the provider's build is not done")
|
||||
fpOf := func(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||
toolOld := fpOf('1')
|
||||
for module, row := range map[string]struct {
|
||||
fp, digest string
|
||||
}{"db": {fpOf('2'), dbDigest}, "app": {fpOf('3'), appDigest}, "tool": {fpOf('4'), toolDigest}} {
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: row.fp, Manifest: row.digest, Repository: "novox/" + module, Commit: strings.Repeat("2", 40), Path: delivery.RootPath})
|
||||
b.putJSON(t, delivery.ModulesBucket, module, mod)
|
||||
fps := []string{row.fp}
|
||||
if module == "tool" {
|
||||
fps = append(fps, toolOld)
|
||||
}
|
||||
for _, fp := range fps {
|
||||
built, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: row.digest})
|
||||
bkey, _ := delivery.BuilderBuildKey(module, fp)
|
||||
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||
gkey, _ := delivery.PartGrantKey(module, fp)
|
||||
grant, _ := delivery.EncodeGrant(delivery.Grant{})
|
||||
b.putJSON(t, delivery.PartGrantsBucket, gkey, grant)
|
||||
}
|
||||
b.place(t, "ace", module)
|
||||
}
|
||||
runsOld, _ := delivery.EncodeHave(delivery.Have{FP: toolOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.putJSON(t, delivery.HaveBucket, "ace.tool", runsOld)
|
||||
if w := b.wantNow(t, "ace", "tool", "the build module's want at the second build", func(w delivery.Want) bool { return w.FP == fpOf('4') }); w.FP != fpOf('4') {
|
||||
t.Errorf("tool's want = %+v", w)
|
||||
}
|
||||
b.wantNow(t, "ace", "db", "the provider's want at its fingerprint", func(w delivery.Want) bool { return w.FP == fpOf('2') })
|
||||
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.ace.app"); err == nil {
|
||||
t.Errorf("app was wanted while its provision was asked: %s", e.Value())
|
||||
}
|
||||
asked(provisions.Provided, "")
|
||||
if w := b.wantNow(t, "ace", "app", "the consumer's want once its provision is provided", func(w delivery.Want) bool { return w.FP == fpOf('3') }); w.FP != fpOf('3') {
|
||||
t.Errorf("app's want = %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
|
||||
|
||||
@@ -28,6 +28,8 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
@@ -60,6 +62,9 @@ var readBuckets = []string{
|
||||
delivery.PresenceBucket,
|
||||
delivery.HaveBucket,
|
||||
provisions.ProvisionsBucket,
|
||||
delivery.SourceBucket,
|
||||
delivery.MergesBucket,
|
||||
graph.Bucket,
|
||||
}
|
||||
|
||||
// readFamilies are the states kept one bucket per node the assigner reads, each named in its manifest's reads: every
|
||||
@@ -140,6 +145,17 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
logf("the bucket %s cannot be opened (%v): the assigner's conditions are said straight to the router until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
if (name == delivery.SourceBucket || name == delivery.MergesBucket) && err != nil {
|
||||
// The gate's source and merges (ADR 0601 §1): without them no release is known to carry a build, so every move of
|
||||
// a record module is asked of a person, as before.
|
||||
logf("the bucket %s cannot be opened (%v): no release's Approve answers a record ask until it is bound", name, err)
|
||||
continue
|
||||
}
|
||||
if name == graph.Bucket && err != nil {
|
||||
// The indexer's module graph (ADR 0614 §2, §3): without it no want waits for a verb it invokes until it is bound.
|
||||
logf("the bucket %s cannot be opened (%v): no want waits for a verb it invokes until it is bound once the bus says it was made", name, err)
|
||||
continue
|
||||
}
|
||||
if name == delivery.PartGrantsBucket && err != nil {
|
||||
// The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound.
|
||||
logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err)
|
||||
@@ -158,7 +174,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
func startsWithout(name string) bool {
|
||||
switch name {
|
||||
case delivery.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket,
|
||||
delivery.AssignerPutsBucket:
|
||||
delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket, graph.Bucket:
|
||||
return true
|
||||
}
|
||||
return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket
|
||||
@@ -241,6 +257,9 @@ var watched = []string{
|
||||
assign.PoliciesBucket,
|
||||
provisions.ProvisionsBucket,
|
||||
delivery.SeatHoldersBucket,
|
||||
delivery.SourceBucket,
|
||||
delivery.MergesBucket,
|
||||
graph.Bucket,
|
||||
}
|
||||
|
||||
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||
@@ -336,8 +355,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
started++
|
||||
var opts []jetstream.WatchOpt
|
||||
if name == delivery.HaveBucket {
|
||||
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
|
||||
if name == delivery.HaveBucket || name == delivery.SourceBucket {
|
||||
// what each node ran healthy before, for a set-back; and the release behind each source revision (ADR 0601 §1)
|
||||
opts = append(opts, jetstream.IncludeHistory())
|
||||
}
|
||||
if err := watch(name, opts...); err != nil {
|
||||
return err
|
||||
|
||||
@@ -155,10 +155,14 @@ func (p *passer) wants(ctx context.Context) {
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
|
||||
grants := p.w.grantsBucket()
|
||||
// The verb edges, once for the pass (ADR 0614 §2, §3).
|
||||
moduleGraph := p.moduleGraph()
|
||||
served := p.served(ctx, get, moduleGraph, nodes)
|
||||
offering := p.offering(ctx, get, manifests, nodes) // bus state (ADR 0614 §5)
|
||||
for name := range names {
|
||||
fp := trunk[name].FP2
|
||||
name := name
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name],
|
||||
Self: func(node string) bool { return placed[node] },
|
||||
Composed: func(node, fp string) (delivery.ComposedDigest, bool) {
|
||||
return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6)
|
||||
@@ -174,7 +178,8 @@ func (p *passer) wants(ctx context.Context) {
|
||||
Allowed: func(node, digest string) bool {
|
||||
key, err := delivery.RecordAskKey(node, name, digest)
|
||||
return err == nil && p.answeredWith(key, assign.OptionYes)
|
||||
}}
|
||||
},
|
||||
Release: func(fp string) (string, bool) { return p.w.release(name, fp) }}
|
||||
if fp != "" {
|
||||
held, _ := delivery.HeldKey(name, fp)
|
||||
grant, _ := delivery.PartGrantKey(name, fp)
|
||||
@@ -191,6 +196,10 @@ func (p *passer) wants(ctx context.Context) {
|
||||
p.clearHeld(name, fp)
|
||||
}
|
||||
}
|
||||
if served != nil {
|
||||
m.Invokes, m.Served = keptInvokes(moduleGraph, name), served
|
||||
}
|
||||
m.Offering = offering
|
||||
last := map[string]string{}
|
||||
for _, node := range m.Nodes {
|
||||
last[node] = p.w.lastHealthy(node, name, fp)
|
||||
@@ -819,7 +828,9 @@ func (p *passer) backupAsk(ctx context.Context, key, node, module string, want d
|
||||
// yeses writes a placed node's wants naming the build alone, and sees to the moves a node that composes its own judged a
|
||||
// person must allow (ADR 0547 §3): for a have of phase
|
||||
// waiting-yes it raises the record ask once, naming the composition waiting and the node's judgement, and on a yes writes
|
||||
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again.
|
||||
// the want again naming it; a yes once written is kept while the want stays at its build, so the node is not asked again. A
|
||||
// move of the build alone (the have says so) to a build an approved release carries is given its yes by that Approve, the
|
||||
// want naming the release, with no ask (ADR 0601 §1).
|
||||
func (p *passer) yeses(ctx context.Context, module string, m assign.Module, before map[string]delivery.Want, out *assign.Outcome) {
|
||||
for _, node := range m.Nodes {
|
||||
if m.Self == nil || !m.Self(node) {
|
||||
@@ -836,25 +847,37 @@ func (p *passer) yeses(ctx context.Context, module string, m assign.Module, befo
|
||||
if w.Removed {
|
||||
continue
|
||||
}
|
||||
yes := ""
|
||||
yes, release := "", ""
|
||||
if had && was.FP == w.FP {
|
||||
yes = was.Yes
|
||||
yes, release = was.Yes, was.Release
|
||||
}
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(p.w.value(delivery.HaveBucket, key)); err == nil && h.Phase == delivery.PhaseWaitingYes && h.Waits != yes {
|
||||
if m.Allowed != nil && m.Allowed(node, h.Waits) {
|
||||
yes = h.Waits
|
||||
} else {
|
||||
switch {
|
||||
case m.Allowed != nil && m.Allowed(node, h.Waits):
|
||||
yes, release = h.Waits, ""
|
||||
case h.BuildAlone && h.FP != w.FP && m.Release != nil:
|
||||
// A move of the build alone, to the build an approved release carries: that Approve is the yes (ADR 0601 §1),
|
||||
// named in the want. A move that changes a setting or a secret too, or one the node could not tell, is asked.
|
||||
if r, ok := m.Release(w.FP); ok {
|
||||
yes, release = h.Waits, r
|
||||
break
|
||||
}
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
default:
|
||||
p.recordAsk(ctx, module, w.FP, assign.RecordNeed{Node: node, Digest: h.Waits, Judgement: h.Judgement, Words: h.Detail})
|
||||
}
|
||||
}
|
||||
// A want carried over from before the node composed its own may still name the controller's composition; a placed
|
||||
// node's want names the build alone (ADR 0547 §2).
|
||||
if yes != w.Yes || w.Composed != "" || !w.Self {
|
||||
w.Yes, w.Composed, w.Self = yes, "", true
|
||||
if yes == "" {
|
||||
release = ""
|
||||
}
|
||||
if yes != w.Yes || release != w.Release || w.Composed != "" || !w.Self {
|
||||
w.Yes, w.Release, w.Composed, w.Self = yes, release, "", true
|
||||
out.Wants[node] = w
|
||||
}
|
||||
}
|
||||
@@ -1071,6 +1094,26 @@ func (p *passer) seatHolders(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// offered says whether some manifest of the trunk, or a module the mesh provides itself, offers a provision under its name
|
||||
// (a module offers its own name too, catalogue.Manifest.Offers): what a requirement nobody offers is told from.
|
||||
func offered(manifests map[string]catalogue.Manifest) func(string) bool {
|
||||
names := map[string]bool{}
|
||||
add := func(m catalogue.Manifest) {
|
||||
for _, o := range m.Offers() {
|
||||
names[o] = true
|
||||
}
|
||||
}
|
||||
for _, m := range manifests {
|
||||
add(m)
|
||||
}
|
||||
for _, module := range delivery.ProvidedModules() {
|
||||
if m, _, err := delivery.ProvidedManifest(module); err == nil {
|
||||
add(m)
|
||||
}
|
||||
}
|
||||
return func(provision string) bool { return names[provision] }
|
||||
}
|
||||
|
||||
// answering is, by provision, the manifests of the modules placed on some node that provide it: what a want's
|
||||
// provisions are judged by (catalogue.Awaited, issue 774). A provider in the catalogue placed nowhere answers nobody.
|
||||
func answering(manifests map[string]catalogue.Manifest, n assign.Nodes) func(string) []catalogue.Manifest {
|
||||
|
||||
@@ -17,6 +17,8 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
@@ -111,6 +113,14 @@ func TestAHoldAskedAgainAfterALiftAndARestartHasANewID(t *testing.T) {
|
||||
// 1906/H4: an answer counts only for the asking it answered: an answer to an earlier hold of the same fingerprint, or to an ask
|
||||
// taken away, is not taken for the hold that stands now.
|
||||
func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
// ADR 0603 §5: a provision is offered by a manifest's provides, by a module's own name, and by a module the mesh provides
|
||||
// itself (mesh-addressing, mesh-wireguard's); nothing else.
|
||||
off := offered(map[string]catalogue.Manifest{"node-tools": {Module: "node-tools"}, "db": {Module: "db", Provides: []catalogue.Offer{{Name: "database"}}}})
|
||||
for provision, want := range map[string]bool{"node-tools": true, "database": true, "mesh-addressing": true, "queue": false} {
|
||||
if off(provision) != want {
|
||||
t.Errorf("offered(%s) = %v, want %v", provision, !want, want)
|
||||
}
|
||||
}
|
||||
key, _ := delivery.HoldAskKey("web", fpTest)
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
put := func(asking int) string {
|
||||
@@ -140,6 +150,23 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
t.Error("an answer to an ask taken away was taken")
|
||||
}
|
||||
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
|
||||
// ADR 0614 §4: the rules read what each node's engine states it reads from its record in `<node>.node`; a node with no
|
||||
// record, or one that states nothing, is read as stating nothing (the rules then read the baseline).
|
||||
t.Run("the rules read what each node's engine reads", func(t *testing.T) {
|
||||
w := newWorld()
|
||||
for node, reads := range map[string][]string{"ace": {"word:module", "word:tool-runner"}, "g14": nil} {
|
||||
raw, err := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, ManifestReads: reads}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveNodeKey(node)
|
||||
w.put(delivery.HaveBucket, key, raw)
|
||||
}
|
||||
n := w.nodes(nil, nil)
|
||||
if !slices.Equal(n.Reads["ace"], []string{"word:module", "word:tool-runner"}) || len(n.Reads["g14"]) != 0 || len(n.Reads["novox"]) != 0 {
|
||||
t.Errorf("the nodes' reads are %v", n.Reads)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// memEntry is a value a memKV holds, as a bucket's Get answers it.
|
||||
@@ -380,6 +407,8 @@ func putOfTheBus(t *testing.T) {
|
||||
func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
// A pair only one source names is a condition of the same kind (issue 1126): folded here to hold the count.
|
||||
aPlacementAloneNamesWhereModulesRun(t)
|
||||
// The verb edges a want waits on come from the graph and what runs (ADR 0614 §2, §3): folded here to hold the count.
|
||||
aWantWaitsOnTheVerbEdgesTheGraphKeeps(t)
|
||||
t.Run("gathers", testThePassGathersTheModulesOnNeitherList)
|
||||
p := newPasser(buckets{}, time.Date(2026, 10, 25, 15, 0, 0, 0, time.UTC))
|
||||
var said []asks.Condition
|
||||
@@ -414,10 +443,11 @@ func TestAModuleOnNeitherListIsRaisedOnceAndClearedOnce(t *testing.T) {
|
||||
if last := logged[len(logged)-1]; last != "screen-lock on g14 has a want again" {
|
||||
t.Errorf("the clear said %q", last)
|
||||
}
|
||||
// 1945 (the bus bar): a module that leaves neither list with no want is cleared in words saying so, never "has a want again".
|
||||
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Bus: true, Words: "the bus moves only by the operator's put"}}}
|
||||
// 1945: a module that leaves neither list with no want is cleared in words saying so, never "has a want again". The bus is
|
||||
// on neither list by a rule as any module (ADR 0601 §1), nothing for the operator to do.
|
||||
bus := map[string]neitherOne{"novox.nats": {node: "novox", module: "nats", holding: assign.Holding{Rule: delivery.RuleGrant, Words: "no credential"}}}
|
||||
p.sayNeither(bus, nil, nil)
|
||||
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NeedsYou) || !strings.Contains(c.Explanation, "the bus moves only by the operator's put") {
|
||||
if c := said[len(said)-1]; c.State != asks.StateRaised || !strings.HasPrefix(c.Explanation, asks.NothingToDo) || !strings.Contains(c.Explanation, "rule 6 holds its first want back: no credential") {
|
||||
t.Errorf("the bus's neither condition = %+v", c)
|
||||
}
|
||||
p.sayNeither(map[string]neitherOne{}, nil, map[string]string{"novox.nats": "it is held, and its held condition says so"})
|
||||
@@ -474,7 +504,6 @@ func testThePassGathersTheModulesOnNeitherList(t *testing.T) {
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Rule: delivery.RuleGrant, Words: "no credential"}}}, "rofi on shanks gets no want: rule 6: no credential"},
|
||||
{nil, wanted, "rofi on shanks has a want"},
|
||||
{map[string]delivery.Want{"shanks": wanted.Wants["shanks"]}, assign.Outcome{}, ""},
|
||||
{nil, assign.Outcome{Rules: map[string]assign.Holding{"shanks": {Bus: true, Words: "the bus moves only by the operator's put"}}}, "rofi on shanks gets no want: the bus moves only by the operator's put"},
|
||||
} {
|
||||
logged = nil
|
||||
p.sayNoWant("rofi", []string{"shanks"}, c.before, c.out)
|
||||
@@ -788,6 +817,74 @@ func mustKey(key string, err error) string {
|
||||
func TestARecordAskOfAHeldBackNodeSurvivesPassesAndItsYesMovesTheWant(t *testing.T) {
|
||||
// A placed node's yes comes from its have (issue 1126): folded here to hold the count.
|
||||
aPlacedNodesWantsNameTheBuildAloneAndItsYesComesFromItsHave(t)
|
||||
aReleasesApproveAnswersTheMoveOfItsBuildAlone(t)
|
||||
}
|
||||
|
||||
// ADR 0601 §1: a placed node's have waiting for a yes to a move of the build alone, to a build an approved release carries, is
|
||||
// given its yes by that Approve, the want naming the release, and no record ask is raised; a move that changes a setting or a
|
||||
// secret too, one at the build it runs, and one no approved release carries are asked of a person as before.
|
||||
func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) {
|
||||
fpOld := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
waiting, was := "sha256:"+strings.Repeat("e", 64), "sha256:"+strings.Repeat("d", 64)
|
||||
commit := strings.Repeat("1", 40)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
alone bool
|
||||
at string // the fingerprint the have runs
|
||||
mergedAt string // the commit the release merged web at; "" for no merge of the release
|
||||
fpSrc uint64 // the source revision at which web's fingerprint last changed
|
||||
yes, asked bool
|
||||
}{
|
||||
{"the build alone, its release approved", true, fpOld, commit, 7, true, false},
|
||||
{"a setting changed too", false, fpOld, commit, 7, false, true},
|
||||
{"a setting changed at the build it runs", true, fpTest, commit, 7, false, true},
|
||||
{"no approved release carries it", true, fpOld, "", 7, false, true},
|
||||
{"the release merged another commit", true, fpOld, strings.Repeat("2", 40), 7, false, true},
|
||||
// Its fingerprint last moved at another release's revision, which merged nothing of web: that Approve answers nothing.
|
||||
{"a fingerprint another release moved", true, fpOld, commit, 8, false, true},
|
||||
// Review rounds 1 and 2: moved by a recompute after the release (a toolchain, a package, a base) with no merge, the
|
||||
// indexer writes fp-src 0, not known, though Src is still the release's: asked of a person.
|
||||
{"a fingerprint moved by a recompute after the release", true, fpOld, commit, 0, false, true},
|
||||
} {
|
||||
asksKV := &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(buckets{delivery.AssignerAsksBucket: asksKV}, time.Now())
|
||||
p.logf = t.Logf
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FPSrc: c.fpSrc, Path: delivery.RootPath})
|
||||
p.w.put(delivery.ModulesBucket, "web", mod)
|
||||
src, _ := delivery.EncodeSource(delivery.Source{Group: "feat/web", Heads: map[string]string{"novox/web": commit}})
|
||||
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, src, 7, time.Now())
|
||||
later, _ := delivery.EncodeSource(delivery.Source{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}})
|
||||
p.w.putEntry(delivery.SourceBucket, delivery.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release
|
||||
if c.mergedAt != "" {
|
||||
merge, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40)}, State: delivery.MergeMerged, Commit: c.mergedAt}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.MergesBucket, "feat/web", merge)
|
||||
other, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/other", Ask: "merge.feat_other.0123456789ab", Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/x", Number: 4, Head: strings.Repeat("4", 40)}, State: delivery.MergeMerged, Commit: commit}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(delivery.MergesBucket, "feat/other", other)
|
||||
}
|
||||
have, _ := delivery.EncodeHave(delivery.Have{FP: c.at, Composed: was, Phase: delivery.PhaseWaitingYes, Health: delivery.HealthOK,
|
||||
Judgement: delivery.JudgedRecreates, Waits: waiting, Detail: "it recreates web.server", BuildAlone: c.alone})
|
||||
p.w.put(delivery.HaveBucket, "g14.web", have)
|
||||
m := assign.Module{Name: "web", FP: fpTest, Nodes: []string{"g14"}, Self: func(string) bool { return true },
|
||||
Release: func(fp string) (string, bool) { return p.w.release("web", fp) }}
|
||||
out := assign.Outcome{Wants: map[string]delivery.Want{"g14": {FP: fpTest, Self: true}}}
|
||||
p.yeses(context.Background(), "web", m, map[string]delivery.Want{"g14": {FP: fpOld, Self: true}}, &out)
|
||||
w := out.Wants["g14"]
|
||||
if got := w.Yes == waiting && w.Release == "feat/web"; got != c.yes || (!c.yes && (w.Yes != "" || w.Release != "")) {
|
||||
t.Errorf("%s: the want is %+v", c.name, w)
|
||||
}
|
||||
akey, _ := delivery.RecordAskKey("g14", "web", waiting)
|
||||
if _, there := asksKV.values[akey]; there != c.asked {
|
||||
t.Errorf("%s: a record ask raised %v, want %v", c.name, there, c.asked)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 774/S26/AC1 (ADR 0549 §5, ADR 0567 §8): the seat's placement alone names where modules run: no controller state is read, so a
|
||||
@@ -928,3 +1025,61 @@ func TestAnApprovedPlacementCallsNoControllerVerb(t *testing.T) {
|
||||
t.Fatalf("the old conditions cleared = %+v", conds)
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0614 §2, §3: of what a module invokes, the pass keeps as edges the verbs the graph does not leave out and another
|
||||
// module serves; a verb is served once a running healthy build's manifest (the builder's build, the indexer's manifest)
|
||||
// serves it, and a build whose manifest cannot be read counts what the graph says its module serves. No graph read: no wait.
|
||||
func aWantWaitsOnTheVerbEdgesTheGraphKeeps(t *testing.T) {
|
||||
p := newPasser(buckets{}, time.Now())
|
||||
fp := func(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||
if p.served(context.Background(), nil, graph.Graph{}, assign.Nodes{}) != nil {
|
||||
t.Error("with no graph read, a want waits on a verb")
|
||||
}
|
||||
for module, n := range map[string]graph.Node{
|
||||
"caller": {FP: fp("c"), Repository: "novox/caller", Invokes: []string{"seat:x.v", "loop.l", "none.t", "caller.own"}, LeftOut: []string{"loop.l"},
|
||||
Serves: []string{"caller.own"}},
|
||||
"server": {FP: fp("e"), Repository: "novox/server", Serves: []string{"seat:x.v", "seat:x.w"}},
|
||||
"loop": {FP: fp("1"), Repository: "novox/loop", Serves: []string{"loop.l"}},
|
||||
"late": {FP: fp("2"), Repository: "novox/late", Serves: []string{"late.t"}},
|
||||
} {
|
||||
raw, err := graph.Encode(n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(graph.Bucket, module, raw)
|
||||
}
|
||||
g := p.moduleGraph()
|
||||
if got := keptInvokes(g, "caller"); !slices.Equal(got, []string{"seat:x.v"}) || keptInvokes(g, "unknown") != nil {
|
||||
t.Errorf("the kept verb edges of caller = %v", got)
|
||||
}
|
||||
digest := strings.Repeat("d", 64)
|
||||
man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "server", JSON: `{"module":"server","claims":[{"name":"x","scope":"mesh","serves":["w"]}]}`})
|
||||
get := func(_ context.Context, key string) ([]byte, error) {
|
||||
if key == digest {
|
||||
return man, nil
|
||||
}
|
||||
return nil, errNotThere
|
||||
}
|
||||
build, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest})
|
||||
key, _ := delivery.BuilderBuildKey("server", fp("f"))
|
||||
p.w.put(delivery.BuilderBuildsBucket, key, build)
|
||||
running := delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
at := func(f string) delivery.Have { h := running; h.FP = f; return h }
|
||||
served := p.served(context.Background(), get, g, assign.Nodes{Haves: map[string]map[string]delivery.Have{
|
||||
"g14": {"server": at(fp("f")), "late": at(fp("2"))}}})
|
||||
for v, want := range map[string]bool{"seat:x.w": true, "seat:x.v": false, "late.t": true} {
|
||||
if served(v) != want {
|
||||
t.Errorf("served(%s) = %v, want %v: the running server's manifest serves w alone, late's is not read", v, served(v), want)
|
||||
}
|
||||
}
|
||||
// ADR 0614 §5: bus-state counts as offered only from a healthy build on a node present; the bus module's build on an
|
||||
// offline node offers nothing, and one whose manifest is not read counts its trunk manifest's offer.
|
||||
bus := map[string]catalogue.Manifest{"bus": {Module: "bus", Provides: []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}}}
|
||||
for _, online := range []bool{false, true} {
|
||||
offering := p.offering(context.Background(), get, bus, assign.Nodes{Online: map[string]bool{"n2": online},
|
||||
Haves: map[string]map[string]delivery.Have{"n2": {"bus": at(fp("2"))}}})
|
||||
if offering(graph.BusState) != online {
|
||||
t.Errorf("with the bus module's node online %v, bus-state offered %v", online, offering(graph.BusState))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -385,7 +385,7 @@ func wantsVerbs(t *testing.T) {
|
||||
}
|
||||
manifest("web", "")
|
||||
manifest("photos", `,"upgrade":{"policy":"record","why":"a person checks the albums"}`)
|
||||
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}]`)
|
||||
manifest("nats", `,"provides":[{"name":"mesh-bus","scope":"mesh"}],"upgrade":{"policy":"record","why":"its streams"}`)
|
||||
for node, fp := range map[string]string{"anchor": fpOld, "laptop": fpNew} {
|
||||
st[haveState].values[node+".nats"], _ = delivery.EncodeHave(delivery.Have{FP: fp, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
}
|
||||
@@ -406,7 +406,7 @@ func wantsVerbs(t *testing.T) {
|
||||
if p := policyOf("photos"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule {
|
||||
t.Errorf("photos = %+v", p)
|
||||
}
|
||||
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromBus {
|
||||
if p := policyOf("nats"); p.Policy != catalogue.PolicyRecord || p.From != catalogue.FromModule { // the policy its manifest says (issue 774)
|
||||
t.Errorf("nats = %+v", p)
|
||||
}
|
||||
// compare: each node's own composition is the digest its have names, and the assigner's findings; nothing splits, the
|
||||
@@ -471,8 +471,12 @@ func wantsVerbs(t *testing.T) {
|
||||
len(said) != 1 || !strings.HasPrefix(said[0], "policy-changed policy-changed.web.") {
|
||||
t.Errorf("chosen %+v, asked %v, said %v", c, asked, said)
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err == nil {
|
||||
t.Error("the bus was given a policy")
|
||||
// A person may choose the bus's policy as any module's (ADR 0601 §1: no part treats the module that provides the bus apart).
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "roll", "why": "x"}, "op"); err != nil {
|
||||
t.Errorf("a person's policy for the bus was refused: %v", err)
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "nats", "policy": "default", "why": "back to the module"}, "op"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.doUpgrade(map[string]any{"module": "web", "policy": "roll"}, "op"); err == nil {
|
||||
t.Error("a policy without why was set")
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
)
|
||||
|
||||
// The verb edges a want waits on (novox/hq ADR 0614 §2, §3): the module graph the indexer writes names what each module
|
||||
// invokes and serves and which of its verb edges it leaves out on a cycle (one place decides the cycle); a want waits for
|
||||
// each kept edge until a build serving its verb runs healthy, read from the haves, the builder's builds and the indexer's
|
||||
// manifests, as a provider is counted (ADR 0603 §5).
|
||||
|
||||
// moduleGraph is the module graph as the indexer last wrote it; a key that does not decode is left out, so its module
|
||||
// waits on no verb.
|
||||
func (p *passer) moduleGraph() graph.Graph {
|
||||
g := graph.Graph{}
|
||||
for key, raw := range p.w.read(graph.Bucket) {
|
||||
module, err := graph.ParseKey(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if n, err := graph.Decode(raw); err == nil {
|
||||
g[module] = n
|
||||
}
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// keptInvokes are the verbs of module the graph keeps as edges: each it invokes and does not leave out, that a module
|
||||
// other than itself serves. A verb no module of the graph serves is no edge, so it holds nothing; nil when the graph holds
|
||||
// no node of module.
|
||||
func keptInvokes(g graph.Graph, module string) []string {
|
||||
n, ok := g[module]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, v := range n.Invokes {
|
||||
if slices.Contains(n.LeftOut, v) {
|
||||
continue
|
||||
}
|
||||
if slices.ContainsFunc(g.Servers(v), func(s string) bool { return s != module }) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// served is, for this pass, whether a build serving a verb runs healthy on some node (graph.Served): every running healthy
|
||||
// have's build (the builder's builds key of its module and fingerprint) names its manifest, and the verbs that manifest
|
||||
// serves are served. A healthy build whose manifest cannot be read counts what the graph says its module serves, said in
|
||||
// the log, so a build read late never holds its callers for good; nil when the graph holds nothing, and no want waits on
|
||||
// a verb.
|
||||
func (p *passer) served(ctx context.Context, get getter, g graph.Graph, n assign.Nodes) func(string) bool {
|
||||
if len(g) == 0 {
|
||||
return nil
|
||||
}
|
||||
served, unread := graph.Served(n.Haves, func(module, fp string) ([]string, bool) { return p.servesOf(ctx, get, module, fp) },
|
||||
func(module string) []string { return g[module].Serves })
|
||||
if len(unread) > 0 {
|
||||
p.logf("the manifest of the running build of %v cannot be read, so what the module graph says they serve is counted as served", unread)
|
||||
}
|
||||
return served
|
||||
}
|
||||
|
||||
// offering is, for this pass, whether a build whose manifest offers a provision runs healthy on a node present (ADR 0603
|
||||
// §5, ADR 0614 §5): every running healthy have on an online node, its build's manifest read as servesOf reads it. A
|
||||
// healthy build whose manifest cannot be read counts what its module's trunk manifest offers, so a build read late never
|
||||
// holds a consumer for good.
|
||||
func (p *passer) offering(ctx context.Context, get getter, trunk map[string]catalogue.Manifest, n assign.Nodes) func(string) bool {
|
||||
offered := map[string]bool{}
|
||||
for node, haves := range n.Haves {
|
||||
if !n.Online[node] {
|
||||
continue
|
||||
}
|
||||
for module, h := range haves {
|
||||
if h.Phase != delivery.PhaseRunning || h.Health != delivery.HealthOK {
|
||||
continue
|
||||
}
|
||||
m, ok := p.manifestOf(ctx, get, module, h.FP)
|
||||
if !ok {
|
||||
m = trunk[module]
|
||||
}
|
||||
for _, o := range m.Offers() {
|
||||
offered[o] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return func(provision string) bool { return offered[provision] }
|
||||
}
|
||||
|
||||
// servesOf are the verbs the manifest of module's build at fp serves (graph.ServesOf), and whether it could be read.
|
||||
func (p *passer) servesOf(ctx context.Context, get getter, module, fp string) ([]string, bool) {
|
||||
m, ok := p.manifestOf(ctx, get, module, fp)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
return graph.ServesOf(m), true
|
||||
}
|
||||
|
||||
// manifestOf is the manifest of module's build at fp (the builder's build names its digest in the indexer's manifests),
|
||||
// and whether it could be read.
|
||||
func (p *passer) manifestOf(ctx context.Context, get getter, module, fp string) (catalogue.Manifest, bool) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
raw := p.w.value(delivery.BuilderBuildsBucket, key)
|
||||
if raw == nil {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || b.Manifest == "" {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
got := p.w.manifest(ctx, get, b.Manifest)
|
||||
if got.err != "" {
|
||||
return catalogue.Manifest{}, false
|
||||
}
|
||||
return got.manifest, true
|
||||
}
|
||||
@@ -197,18 +197,14 @@ func (s *wantsTools) doUpgrade(args map[string]any, by string) (any, error) {
|
||||
}
|
||||
|
||||
// setPolicy writes a person's choice of a module's policy, or deletes it for default, records the act in the hand-act log
|
||||
// and says policy-changed. The bus is never rolled: a choice for it is refused.
|
||||
// and says policy-changed. The bus's choice is a person's as any module's (ADR 0601 §1).
|
||||
func (s *wantsTools) setPolicy(module, policy, why, by string, manifests map[string]catalogue.Manifest) (any, error) {
|
||||
if module == "" {
|
||||
return nil, errors.New("a policy is set for one module: name it")
|
||||
}
|
||||
m, ok := manifests[module]
|
||||
if !ok {
|
||||
if _, ok := manifests[module]; !ok {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest of %s", module)
|
||||
}
|
||||
if catalogue.ProvidesBus(m) {
|
||||
return nil, fmt.Errorf("%s is the bus, which is never rolled whoever says so (novox/hq ADR 0236): its step is the put on the operator's warrant", module)
|
||||
}
|
||||
if why == "" {
|
||||
return nil, errors.New("a person's policy says why")
|
||||
}
|
||||
|
||||
@@ -37,6 +37,9 @@ type world struct {
|
||||
// placement is the seat's state placement assembled node by node from its keys in either form (mesh-sdk
|
||||
// delivery.PlacementKeys; research 072 row 37): the one reader of its keys in the assigner.
|
||||
placement *delivery.PlacementKeys
|
||||
// sourceGroups is, by revision of the gate's source.main, the release (group) that moved it, from every revision its
|
||||
// history keeps (watched with history): a build's source revision names the release that carries it (ADR 0601 §1).
|
||||
sourceGroups map[uint64]string
|
||||
}
|
||||
|
||||
// entryMeta is a value's revision in its bucket and when it was written.
|
||||
@@ -54,7 +57,8 @@ type parsed struct {
|
||||
|
||||
func newWorld() *world {
|
||||
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, changes: map[string]uint64{},
|
||||
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys()}
|
||||
meta: map[string]entryMeta{}, trunkBuilt: map[string]delivery.BuildLogEntry{}, placement: delivery.NewPlacementKeys(),
|
||||
sourceGroups: map[uint64]string{}}
|
||||
}
|
||||
|
||||
// putEntry is put, keeping the value's revision and the time it was written.
|
||||
@@ -70,6 +74,44 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t
|
||||
if bucket == delivery.PlacementBucket {
|
||||
_, _ = w.placement.Put(key, value, revision, false) // put took it already; this is its revision
|
||||
}
|
||||
if bucket == delivery.SourceBucket && key == delivery.SourceMainKey {
|
||||
if s, err := delivery.DecodeSource(value); err == nil && s.Group != "" {
|
||||
w.sourceGroups[revision] = s.Group
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// release is the release whose Approve carries module's build fp (ADR 0601 §1), and whether one does: the trunk's value of
|
||||
// the module names fp, the source revision at which that fingerprint last changed (FPSrc, never the Src of its last merge,
|
||||
// which a recompute with no merge keeps) names the release that moved it, and the gate's merge of that release, which it
|
||||
// writes on the operator's Approve, holds the module's own trunk merge among its merged pull requests. Any of them
|
||||
// unknown: none, and the move is asked of a person.
|
||||
func (w *world) release(module, fp string) (string, bool) {
|
||||
t, ok := w.trunk()[module]
|
||||
if !ok || fp == "" || t.FP2 != fp || t.FPSrc == 0 || t.Commit == "" {
|
||||
return "", false
|
||||
}
|
||||
w.mu.Lock()
|
||||
group := w.sourceGroups[t.FPSrc]
|
||||
w.mu.Unlock()
|
||||
key, err := delivery.MergesKey(group)
|
||||
if group == "" || err != nil {
|
||||
return "", false
|
||||
}
|
||||
raw := w.value(delivery.MergesBucket, key)
|
||||
if raw == nil {
|
||||
return "", false
|
||||
}
|
||||
merge, err := delivery.DecodeMerge(raw)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
for _, pull := range merge.Pulls {
|
||||
if pull.State == delivery.MergeMerged && pull.Commit == t.Commit && delivery.SameRepository(pull.Repository, t.Repository) {
|
||||
return group, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// metaOf is the revision and time of the value a bucket holds at key, as the watch said them.
|
||||
@@ -296,7 +338,8 @@ func (w *world) wantsOf() map[string]map[string]delivery.Want {
|
||||
|
||||
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
|
||||
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
|
||||
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{}}
|
||||
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{},
|
||||
Reads: map[string][]string{}}
|
||||
for key, value := range w.read(delivery.PresenceBucket) {
|
||||
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || node == "" {
|
||||
@@ -308,7 +351,14 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
||||
}
|
||||
for key, value := range w.read(delivery.HaveBucket) {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if isNode {
|
||||
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
|
||||
if h, err := delivery.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
n.Reads[node] = h.Record.ManifestReads
|
||||
}
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(value); err == nil {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-assigner
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
@@ -20,3 +20,7 @@ require (
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -119,14 +119,6 @@ func NeitherCondition(node, module string, h Holding, raised bool) asks.Conditio
|
||||
if n := len([]rune(headline)); n > asks.HeadlineLength {
|
||||
headline = string([]rune(headline)[:asks.HeadlineLength])
|
||||
}
|
||||
if h.Bus {
|
||||
needs := fmt.Sprintf("put %s on %s with assigner_put", module, node)
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline, Needs: needs,
|
||||
Explanation: fmt.Sprintf("%s%s. %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
"hold it either, so nothing applies it there. The assigner never moves the bus: %s. This clears once the node has a want for it.",
|
||||
asks.NeedsYou, needs, module, node, h.Words)}
|
||||
}
|
||||
return asks.Condition{Key: key, Severity: asks.SeverityUrgent, Subject: subject, State: asks.StateRaised,
|
||||
Summary: headline, Headline: headline,
|
||||
Explanation: fmt.Sprintf("%s %s is assigned to %s, but the node holds no want for it and the node's own composition does not "+
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
|
||||
// A person's upgrade policy is the assigner's (novox/hq ADR 0565 §7, §8, issue 1953 D4, D5): kept in its state `policies`, one
|
||||
// key per module, set and taken back through `mesh-wants.upgrade`. Every reader takes one derivation, mesh-sdk's
|
||||
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus never rolled.
|
||||
// catalogue.Upgrade: a person's choice over the module's word, its irreplaceable data and the default, the bus as any module (ADR 0601 §1).
|
||||
|
||||
// PoliciesBucket is the assigner's state `policies`.
|
||||
var PoliciesBucket = delivery.AssignerModule + "_" + delivery.AssignerPoliciesState
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package assign
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
)
|
||||
|
||||
// verbWaiting says why the module's want on node waits for a verb it invokes (novox/hq ADR 0614 §2): the first seat verb or
|
||||
// module tool of Invokes (the module graph's verb edges, those it leaves out on a cycle not among them, §3) that no
|
||||
// running healthy build serves. Nil Served (nothing read of what runs) waits for none, and a verb a running build serves
|
||||
// already adds no wait. It holds under the provisions' rule (RuleProvisions): no reader knows a rule number of its own
|
||||
// yet, so the words name the edge (graph.VerbWaitWords, which the delivery view reads).
|
||||
func (m Module) verbWaiting(node string) (string, bool) {
|
||||
if m.Served == nil {
|
||||
return "", false
|
||||
}
|
||||
for _, v := range slices.Sorted(slices.Values(m.Invokes)) {
|
||||
if !m.Served(v) {
|
||||
return graph.VerbWaitWords(v, m.Name, node), true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// busStateWaiting says why the module's want on node waits for bus state (ADR 0614 §5): its manifest asks bus state (a
|
||||
// bus-state entry, as the module graph reads it) and no build whose manifest offers bus-state runs healthy on a node
|
||||
// present. The provision bus-state is answered with no credential of a consumer's own, so ADR 0603 §5 counts the
|
||||
// provider's own have and no provisioner key is waited for, handed or not. Nil Offering (nothing read of what runs) waits
|
||||
// for none, and neither does a module that offers bus-state itself. It holds under the provisions' rule, its words naming bus state (graph.BusStateWaitWords).
|
||||
func (m Module) busStateWaiting(node string) (string, bool) {
|
||||
if len(m.Manifest.BusState) == 0 || m.Offering == nil || m.Offering(graph.BusState) ||
|
||||
slices.Contains(m.Manifest.Offers(), graph.BusState) { // the bus module offering it waits on nobody for it
|
||||
|
||||
return "", false
|
||||
}
|
||||
return graph.BusStateWaitWords(m.Name, node), true
|
||||
}
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
)
|
||||
|
||||
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
|
||||
@@ -36,6 +38,21 @@ type Module struct {
|
||||
// Answering is the manifests of the modules placed somewhere that provide a provision: what catalogue.Awaited
|
||||
// judges a want's provisions by (issue 774). Nil: none known, and no key is waited for that is not there.
|
||||
Answering func(provision string) []catalogue.Manifest
|
||||
// Offered says whether any module offers a provision by name or by its own name (catalogue.Manifest.Offers) among every
|
||||
// manifest the trunk holds and the modules the mesh provides itself, placed or not. Nil: not known, and no requirement
|
||||
// is waited on for want of an offer.
|
||||
Offered func(provision string) bool
|
||||
// Invokes are the seat verbs and module tools the module graph says the module invokes and keeps as edges to the
|
||||
// modules serving them, those it leaves out on a cycle not among them (ADR 0614 §2, §3): the want waits until each is
|
||||
// served (Served). Nil: no graph node is read, and no want waits on a verb.
|
||||
Invokes []string
|
||||
// Served says whether a build serving a seat verb or module tool runs healthy, counted from its holder's have (ADR 0614
|
||||
// §2, as ADR 0603 §5 counts a provider's). Nil: nothing is read of what runs, and no want waits on a verb.
|
||||
Served func(invoke string) bool
|
||||
// Offering says whether a build whose manifest offers a provision runs healthy on a node present, counted from the
|
||||
// provider's own have (ADR 0603 §5): what a want asking bus state waits for (ADR 0614 §5). Nil: nothing is read of what
|
||||
// runs, and no want waits on bus state.
|
||||
Offering func(provision string) bool
|
||||
// Chosen is a person's choice of the module's upgrade policy (the assigner's `policies`, ADR 0565 §7); nil: none.
|
||||
Chosen *catalogue.PersonPolicy
|
||||
Nodes []string
|
||||
@@ -45,6 +62,10 @@ type Module struct {
|
||||
// Allowed says whether the operator answered yes to the record ask about the move of this module on node to the composition
|
||||
// with this digest (ADR 0539 §14). Nil: none did.
|
||||
Allowed func(node, digest string) bool
|
||||
// Release names the release whose Approve carries the build fp, as mesh-delivery's gate holds it for the release's heads,
|
||||
// read by the build's source revision (ADR 0601 §1): that Approve is the yes to a move of the build alone. False when no
|
||||
// approved release is known to carry it. Nil: none is read.
|
||||
Release func(fp string) (string, bool)
|
||||
// Released is a hold on FP the operator released (ADR 0539 §19): FP is not held again by a health check that failed.
|
||||
Released bool
|
||||
// Restored says whether the operator answered "restore" to the restore ask of this data module on node: it is then set back
|
||||
@@ -56,6 +77,9 @@ type Module struct {
|
||||
Now time.Time
|
||||
// Puts are the puts of the module, by node (ADR 0546): while one owns a node's want (PutOwns), the rules pass over it.
|
||||
Puts map[string]delivery.Put
|
||||
// Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR
|
||||
// 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none.
|
||||
Needs []string
|
||||
// Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one
|
||||
// source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while
|
||||
// the node was placed does not outlive the placement (issue 1126). Nil: no node does.
|
||||
@@ -77,6 +101,9 @@ type Nodes struct {
|
||||
Haves map[string]map[string]delivery.Have
|
||||
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
|
||||
Providers map[string]map[string][]string
|
||||
// Reads is, by node, the reader capabilities its node-engine states in its node record (delivery.NodeRecord.ManifestReads,
|
||||
// ADR 0614 §4). A node with none, or none read, is read as stating the baseline (catalogue.StatedManifestReads).
|
||||
Reads map[string][]string
|
||||
}
|
||||
|
||||
// healthy says whether a have runs fp healthy.
|
||||
@@ -130,15 +157,36 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) {
|
||||
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s on %s for %s", m.Name, node,
|
||||
short(m.FP))
|
||||
}
|
||||
if words, lacking := m.readerLacking(node, n); lacking {
|
||||
return false, delivery.RuleComposed, words
|
||||
}
|
||||
if _, ok := m.composition(node, m.FP); !ok {
|
||||
return false, delivery.RuleComposed, fmt.Sprintf("%s has named no composition of its own for %s at %s yet", node, m.Name, short(m.FP))
|
||||
}
|
||||
if words, waiting := m.provisionWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
}
|
||||
if words, waiting := m.verbWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
}
|
||||
if words, waiting := m.busStateWaiting(node); waiting {
|
||||
return false, delivery.RuleProvisions, words
|
||||
}
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
// readerLacking says why the module's want on node is held for the node's engine (ADR 0614 §4): a reader capability its
|
||||
// manifest needs that the engine does not state, read as the baseline when it states none. It is held under rule 7 (the
|
||||
// composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the
|
||||
// engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states.
|
||||
func (m Module) readerLacking(node string, n Nodes) (string, bool) {
|
||||
missing := catalogue.MissingManifestReads(m.Needs, n.Reads[node])
|
||||
if len(missing) == 0 {
|
||||
return "", false
|
||||
}
|
||||
return graph.ReaderWaitWords(node, missing, m.Name, short(m.FP)), true // the words the delivery view names the edge by
|
||||
}
|
||||
|
||||
// provisionWaiting says why the module's want on node waits for a provision (ADR 0570 §4): a provision it requires that has
|
||||
// no key yet, or the first of its keys that is not provided. Nil Provisions (the seat's state not on the bus) waits for none.
|
||||
func (m Module) provisionWaiting(node string) (string, bool) {
|
||||
@@ -149,8 +197,36 @@ func (m Module) provisionWaiting(node string) (string, bool) {
|
||||
return fmt.Sprintf("provision %s has no key yet for %s on %s", missing[0], m.Name, node), true
|
||||
}
|
||||
keyed := m.Provisions(node)
|
||||
// Only a provision the newest manifest still requires is waited on (ADR 0603 §9): a key of an older revision's edge,
|
||||
// asked or retiring for the operator's answer (ADR 0570 §5), holds nothing. A requirement catalogue.Awaited leaves out is
|
||||
// not waited on when something offers it (a module's own name, a module the mesh provides itself, a provision the
|
||||
// composition does not key: optional, a toolchain on the module's own machine, a login the controller makes). One that
|
||||
// nothing offers is waited on (ADR 0603 §5: an unknown provider has provided nothing), so a consumer never moves ahead of
|
||||
// a provider whose module the indexer has not yet taken in; it holds loudly, since the watchman raises the hold past its
|
||||
// bound (ADR 0603 §8).
|
||||
needed := m.Manifest.Requires
|
||||
if m.Answering != nil {
|
||||
needed = catalogue.Awaited(m.Manifest, m.Answering)
|
||||
have := map[string]bool{}
|
||||
for _, p := range keyed {
|
||||
have[p.Provision] = true
|
||||
}
|
||||
for _, r := range slices.Sorted(slices.Values(m.Manifest.Requires)) {
|
||||
if have[r] && !m.Manifest.OptionalRequires[r] && !slices.Contains(needed, r) {
|
||||
needed = append(needed, r) // the provisioner keys it from the manifests, whoever is placed yet
|
||||
continue
|
||||
}
|
||||
if m.Offered == nil || m.Manifest.OptionalRequires[r] || len(m.Answering(r)) > 0 || m.Offered(r) || slices.Contains(needed, r) {
|
||||
continue
|
||||
}
|
||||
needed = append(needed, r)
|
||||
if !have[r] {
|
||||
return fmt.Sprintf("no module offers %s yet for %s on %s: an unknown provider has provided nothing, so the want waits for one, and the watchman raises it past its bound", r, m.Name, node), true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(keyed)) {
|
||||
if p := keyed[key]; p.State != provisions.Provided {
|
||||
if p := keyed[key]; p.State != provisions.Provided && slices.Contains(needed, p.Provision) {
|
||||
words := fmt.Sprintf("the provision %s of %s on %s is %s, not provided", key, m.Name, node, p.State)
|
||||
if p.Why != "" {
|
||||
words += ": " + p.Why
|
||||
@@ -268,29 +344,18 @@ type RecordNeed struct {
|
||||
Words string
|
||||
}
|
||||
|
||||
// Holding is a rule holding a node back, and why. Bus marks the bar on the bus instead of one of the seven rules (Rule is then
|
||||
// 0): the bus moves only by the operator's put (ADR 0539 §15, ADR 0546 §8), never by the assigner (issue 1945: nats on novox,
|
||||
// running an old-form build, got no want and the pass said nothing).
|
||||
// Holding is a rule holding a node back, and why. The bus module is held back by the rules alone, as every module (ADR 0601 §1,
|
||||
// §2): no bar of its own.
|
||||
type Holding struct {
|
||||
Rule int
|
||||
Words string
|
||||
Bus bool
|
||||
}
|
||||
|
||||
// Reason is the holding in words: the rule and its words, or for the bus bar its words alone.
|
||||
// Reason is the holding in words: the rule and its words.
|
||||
func (h Holding) Reason() string {
|
||||
if h.Bus {
|
||||
return h.Words
|
||||
}
|
||||
return fmt.Sprintf("rule %d: %s", h.Rule, h.Words)
|
||||
}
|
||||
|
||||
// BusBarWords say why the assigner writes no want moving the bus on node: it runs prev, the trunk builds fp, and only the
|
||||
// operator's put moves it (ADR 0539 §15, ADR 0546 §8). The words reach the operator, so they name no record.
|
||||
func BusBarWords(node, prev, fp string) string {
|
||||
return fmt.Sprintf("only your put moves the bus: %s runs %s, the trunk builds %s; put it with assigner_put", node, short(prev), short(fp))
|
||||
}
|
||||
|
||||
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
|
||||
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
|
||||
@@ -425,14 +490,6 @@ func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[st
|
||||
if need, wait := at.mustAsk(node, asked, hadLive, ran, running.FP, running.Composed == "", d); wait {
|
||||
if need != nil {
|
||||
out.Record = append(out.Record, *need)
|
||||
} else if prev, barred := at.busBar(asked, hadLive, running.FP); barred {
|
||||
// The bus is barred: no want moves it but the operator's put. Said, never silent (issue 1945): its holding is named,
|
||||
// and a node that runs it with no live want has it on neither list, so its condition stays raised.
|
||||
h := Holding{Bus: true, Words: BusBarWords(node, prev, at.FP)}
|
||||
out.Rules[node] = h
|
||||
if !hadLive && running.Phase != delivery.PhaseRemoved && !held {
|
||||
out.Neither[node] = h
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -659,10 +716,10 @@ func short(fp string) string {
|
||||
return part[:12]
|
||||
}
|
||||
|
||||
// mustAsk says whether the move to the composition d on node waits: for the operator's yes (need names the ask) or, for the bus,
|
||||
// for the operator's hand (ADR 0539 §14, §15). A module the node never ran (it has no want and no have), a move judged neither, a module that is not of
|
||||
// policy record and a want that stays are written as any want; so is a revision of the bus module's composition at the
|
||||
// same fingerprint judged neither (a user list only, §7).
|
||||
// mustAsk says whether the move to the composition d on node waits for the operator's yes (need names the ask, ADR 0539 §14). A
|
||||
// module the node never ran (it has no want and no have), a move judged neither, a module that is not of policy record and a want
|
||||
// that stays are written as any want. The bus module is no exception: it moves by the rules (ADR 0601 §1), its node backing up its
|
||||
// streams before it applies (§2); the assigner asks no snapshot for it.
|
||||
func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, running string, haveNamesNoComposition bool, d delivery.ComposedDigest) (*RecordNeed, bool) {
|
||||
if (!had && !ran) || before.Removed || (before.FP == m.FP && before.Composed == d.Digest) {
|
||||
return nil, false
|
||||
@@ -674,9 +731,6 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
if !had && ran && running == m.FP && haveNamesNoComposition && d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
if _, barred := m.busBar(before, had, running); barred {
|
||||
return nil, true // the bus is never moved by the assigner; the operator writes its want (§15)
|
||||
}
|
||||
if !m.recorded(d) || d.Judgement == delivery.JudgedNeither || d.Judgement == delivery.JudgedUnchanged {
|
||||
return nil, false
|
||||
}
|
||||
@@ -686,17 +740,7 @@ func (m Module) mustAsk(node string, before delivery.Want, had, ran bool, runnin
|
||||
return &RecordNeed{Node: node, Digest: d.Digest, Judgement: d.Judgement, Words: d.Words}, true
|
||||
}
|
||||
|
||||
// busBar says whether m is the bus and the move would take it off the fingerprint it ran before, prev: its want's, or with no
|
||||
// readable want the one its have names (ADR 0539 §15 bars a new one).
|
||||
func (m Module) busBar(before delivery.Want, had bool, running string) (prev string, barred bool) {
|
||||
prev = before.FP
|
||||
if !had {
|
||||
prev = running
|
||||
}
|
||||
return prev, catalogue.ProvidesBus(m.Manifest) && prev != m.FP
|
||||
}
|
||||
|
||||
// recorded says whether the module is of policy record: a person's choice at the assigner over everything but the bus (ADR
|
||||
// recorded says whether the module is of policy record: a person's choice at the assigner over the manifest's word (ADR
|
||||
// 0565 §7, catalogue.Upgrade); for a module with none, as the controller holds it beside the composition's digest until the
|
||||
// lease write (ADR 0539 §14); on a composition made by hand, which carries none, as the manifest derives it.
|
||||
func (m Module) recorded(d delivery.ComposedDigest) bool {
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/provisions"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
)
|
||||
|
||||
func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||
@@ -104,6 +106,111 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
t.Errorf("provided provisions held g14's want: %+v (%s)", out.Wants["g14"], out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// ADR 0614 §4: a want is held on a node whose engine does not state a reader capability the manifest needs, under rule 7
|
||||
// with the capability as its words, and moves once the engine states it; an engine stating nothing reads the baseline,
|
||||
// so a manifest of the baseline's words, the node-engine's own new build among them, is not held.
|
||||
hostNeeds, err := catalogue.ManifestNeeds([]byte(`{"module":"mesh-host","version":"1","slug":"host","build":{"artifacts":[` +
|
||||
`{"name":"host-arch","kind":"bundle","language":"go","system":"arch","from":"cmd/mesh-host"}]},"consumes":["restic.backed-up"],` +
|
||||
`"state":[{"name":"health"}],"config":{"x":{"type":"number","minimum":1,"default":50,"runtime":true}},` +
|
||||
`"resources":[{"id":"k","type":"file","path":"/var/lib/k","content":"${config:x}\n"}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
why string
|
||||
name string
|
||||
needs []string
|
||||
reads map[string][]string
|
||||
held bool
|
||||
}{
|
||||
{"a word the baseline lacks, on engines stating nothing", "mail", []string{"word:module", "word:tool-runner"}, nil, true},
|
||||
{"the same, once both engines state it", "mail", []string{"word:module", "word:tool-runner"},
|
||||
map[string][]string{"ace": catalogue.ManifestReads(), "g14": catalogue.ManifestReads()}, false},
|
||||
{"the baseline's words on engines stating nothing", "mail", []string{"word:claims.serves", "seat:mesh-catalog", "form:config"}, nil, false},
|
||||
{"the node-engine's new build, whose contract has more words than its manifest uses", "mesh-host", hostNeeds, nil, false},
|
||||
} {
|
||||
m, n := ready()
|
||||
m.Name, m.Needs, n.Reads = c.name, c.needs, c.reads
|
||||
n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"]
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if moved := out.Wants["g14"].FP == fNew; moved == c.held {
|
||||
t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"])
|
||||
}
|
||||
if r := out.Rules["g14"]; c.held && (r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, "word:tool-runner") ||
|
||||
!strings.Contains(r.Words, "node-engine of g14")) {
|
||||
t.Errorf("%s: held by %+v, want rule 7 naming the capability", c.why, r)
|
||||
}
|
||||
if w := out.Wants["g14"]; c.held && (w.Rule != delivery.RuleComposed || w.Newest != fNew) {
|
||||
t.Errorf("%s: g14's want %+v, want it kept at what it runs and naming rule 7", c.why, w)
|
||||
}
|
||||
}
|
||||
// ADR 0614 §2, §3: a want waits, under the provisions' rule, until each verb it invokes is served by a running build,
|
||||
// and moves once one is; a verb a running build serves already adds no wait, and an edge the graph leaves out on a cycle
|
||||
// is not among Invokes, so it holds nothing; nothing read of what runs waits for none.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
invokes []string
|
||||
served []string
|
||||
nilRead bool
|
||||
holds string
|
||||
}{
|
||||
{"a verb no running build serves", []string{"store.store_put", "seat:mesh-catalog.modules"}, []string{"store.store_put"}, false, "seat:mesh-catalog.modules"},
|
||||
{"once a build serving it runs", []string{"seat:mesh-catalog.modules"}, []string{"seat:mesh-catalog.modules"}, false, ""},
|
||||
{"an edge left out on a cycle", nil, nil, false, ""},
|
||||
{"nothing read of what runs", []string{"seat:mesh-catalog.modules"}, nil, true, ""},
|
||||
// ADR 0614 §5: a module asking bus state is held while no build offering bus-state runs, and moves once one does.
|
||||
{"bus state no running build offers", nil, nil, false, graph.BusState},
|
||||
{"bus state a running build offers", nil, []string{graph.BusState}, false, ""},
|
||||
{"bus state its own module offers", nil, nil, false, ""},
|
||||
} {
|
||||
m, n := ready()
|
||||
m.Invokes = c.invokes
|
||||
if !c.nilRead {
|
||||
m.Served = func(v string) bool { return slices.Contains(c.served, v) }
|
||||
}
|
||||
if strings.HasPrefix(c.name, "bus state") {
|
||||
m.Manifest.BusState = []catalogue.BusStateEntry{{Name: "cache", Storage: "key-value"}}
|
||||
m.Offering = func(p string) bool { return slices.Contains(c.served, p) }
|
||||
if c.name == "bus state its own module offers" {
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: graph.BusState, Scope: catalogue.ScopeMesh}}
|
||||
}
|
||||
}
|
||||
words := graph.VerbWaitWords(c.holds, "mail", "g14")
|
||||
if c.holds == graph.BusState {
|
||||
words = graph.BusStateWaitWords("mail", "g14")
|
||||
}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if c.holds == "" {
|
||||
if out.Wants["g14"].FP != fNew {
|
||||
t.Errorf("%s: held %+v", c.name, out.Rules["g14"])
|
||||
}
|
||||
continue
|
||||
}
|
||||
if out.Wants["g14"].FP == fNew || out.Rules["g14"].Rule != delivery.RuleProvisions || out.Rules["g14"].Words != words {
|
||||
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
|
||||
}
|
||||
}
|
||||
// ADR 0603 §9: the newest source revision governs. The module now at fNew no longer requires the database an older revision
|
||||
// asked for and requires oidc instead; the seat's keys follow the newest manifest, so the older edge is no wait: a provided
|
||||
// oidc moves the want, an asked one holds it and names oidc and never database.
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
state, older string
|
||||
moves bool
|
||||
}{{"the replaced edge provided, the older edge retiring", provisions.Provided, provisions.Retiring, true},
|
||||
{"the replaced edge provided, the older edge asked", provisions.Provided, provisions.Asked, true},
|
||||
{"the replaced edge asked, the older edge retiring", provisions.Asked, provisions.Retiring, false}} {
|
||||
m, n := ready()
|
||||
m.Manifest.Requires = []string{"oidc"}
|
||||
m.Provisions = func(node string) map[string]provisions.Provision {
|
||||
return map[string]provisions.Provision{node + ".mail.oidc": {Provision: "oidc", State: c.state},
|
||||
node + ".mail.database": {Provision: "database", State: c.older, Why: "the operator's answer"}}
|
||||
}
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if got := out.Wants["g14"].FP == fNew; got != c.moves || (!c.moves && (!strings.Contains(out.Rules["g14"].Words, "oidc") || strings.Contains(out.Rules["g14"].Words, "database"))) {
|
||||
t.Errorf("%s: want %+v, held by %q", c.name, out.Wants["g14"], out.Rules["g14"].Words)
|
||||
}
|
||||
}
|
||||
// 1954/D1 (the lead, 2026-10-29): a required provision with no key yet holds the want, named; and only one the
|
||||
// composition keys (catalogue.Awaited, issue 774), replaying 2026-10-30: grafana (not handed) waited on ace for
|
||||
// oidc-client, which the controller still makes, and mesh-build-go (handed) for go-toolchain on its own machine.
|
||||
@@ -155,6 +262,24 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// ADR 0603 §5: a requirement nobody offers is an unknown provider and waits, loudly; one something offers (a module's own
|
||||
// name, a module the mesh provides itself) is left to catalogue.Awaited as before and does not.
|
||||
for _, c := range []struct {
|
||||
name, requires string
|
||||
offered bool
|
||||
}{{"a module's own name (node-tools)", "node-tools", true}, {"a module the mesh provides itself (mesh-addressing)", "mesh-addressing", true},
|
||||
{"a provision nobody offers", "database", false}} {
|
||||
m, n := ready()
|
||||
m.Manifest.Requires = []string{c.requires}
|
||||
m.Answering = func(string) []catalogue.Manifest { return nil }
|
||||
m.Offered = func(p string) bool { return c.offered && p == c.requires }
|
||||
m.Provisions = func(string) map[string]provisions.Provision { return map[string]provisions.Provision{} }
|
||||
out := Wants(m, n, wantedOld(), nil)
|
||||
if got := out.Wants["g14"].FP == fNew; got != c.offered || (!c.offered && (out.Rules["g14"].Rule != delivery.RuleProvisions ||
|
||||
!strings.Contains(out.Rules["g14"].Words, "no module offers database yet") || !strings.Contains(out.Rules["g14"].Words, "past its bound"))) {
|
||||
t.Errorf("%s: want %+v, held by %+v", c.name, out.Wants["g14"], out.Rules["g14"])
|
||||
}
|
||||
}
|
||||
// 1960: rule 5 holds no want, and no module is left without a first want for a provider down or a node offline; the
|
||||
// provider down is said (Unreached). novox on 2026-10-29: route-proxy down held de-spiegel and link2pay, minio not
|
||||
// healthy held photos, and novox offline left a module with no canary.
|
||||
@@ -639,11 +764,13 @@ func TestAMoveJudgedNeitherOrNeverRunOrNotRecordedIsWrittenAtOnce(t *testing.T)
|
||||
|
||||
// 1906/D4 (ADR 0539 §7, §15): the bus module's new fingerprint is never written by the assigner, but a revision of its
|
||||
// composition at the same fingerprint that only changes the user list (judged neither) is written at once.
|
||||
func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
func TestTheBusMovesByTheRulesAndItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP == fNew {
|
||||
t.Errorf("the bus moved to a new fingerprint: %+v", out.Wants["ace"])
|
||||
// ADR 0601 §1, §2: the bus module's want is written by the rules, as any module's, and no snapshot is asked for it: its node
|
||||
// backs its streams up before it applies.
|
||||
if out := Wants(m, n, wantedOld(), nil); out.Wants["ace"].FP != fNew || len(out.Record) != 0 || len(out.Neither) != 0 {
|
||||
t.Errorf("the bus did not move by the rules: %+v", out)
|
||||
}
|
||||
running := map[string]delivery.Want{"ace": {FP: fNew, Composed: dig(fOld)}, "g14": {FP: fNew, Composed: dig(fOld)}}
|
||||
out := Wants(m, n, running, nil)
|
||||
@@ -696,23 +823,6 @@ func TestTheBusIsNotMovedButItsUserListRevisionIsWrittenAtOnce(t *testing.T) {
|
||||
t.Errorf("a held bus did not take its user-list revision as a put: %+v", w)
|
||||
}
|
||||
m.Puts = nil
|
||||
// 1945 (the bus bar made loud): a node that runs the bus at an old-form build and holds no want for it gets none, since only
|
||||
// the operator's put moves the bus, and the pass says so: a holding naming the bar, and the node on neither list so its
|
||||
// condition stays raised. A node with a live want keeps it and is not on neither list.
|
||||
m.Composed = composedEverywhere
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: "b-1700000000-oldform", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
out = Wants(m, n, map[string]delivery.Want{"g14": {FP: fOld, Composed: dig(fOld)}}, nil)
|
||||
if w, wrote := out.Wants["ace"]; wrote {
|
||||
t.Errorf("the bus bar wrote ace's want %+v", w)
|
||||
}
|
||||
h, on := out.Neither["ace"]
|
||||
if !on || !h.Bus || out.Rules["ace"] != h || !strings.Contains(h.Words, "only your put moves the bus") || !strings.Contains(h.Words, "ace runs b-1700000000-oldform") ||
|
||||
!strings.Contains(h.Words, "assigner_put") {
|
||||
t.Errorf("the barred bus on ace: neither %+v (%v), rules %+v", h, on, out.Rules["ace"])
|
||||
}
|
||||
if _, on := out.Neither["g14"]; on || out.Wants["g14"].FP == fNew {
|
||||
t.Errorf("g14 has a live want: neither %+v, its want %+v", out.Neither, out.Wants["g14"])
|
||||
}
|
||||
stepsOfAPut(t)
|
||||
}
|
||||
|
||||
@@ -889,8 +999,8 @@ func TestTheRecordPolicyIsTheOneTheControllerHolds(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Review round 1, item 2 (ADR 0539 §15): a node that runs the bus with no readable want compares the new fingerprint with the one
|
||||
// it runs, from its have: the same fingerprint is not a new one (written by the rules), another is barred.
|
||||
// A node that runs the bus with no readable want gets its want by the rules, at the fingerprint it runs or at a new one alike: the
|
||||
// bus has no bar of its own (ADR 0601 §1).
|
||||
func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
|
||||
m, n, _ := movingTo(delivery.JudgedNeither)
|
||||
m.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}}
|
||||
@@ -899,8 +1009,8 @@ func TestTheBusWithNoWantIsComparedWithWhatItRuns(t *testing.T) {
|
||||
t.Errorf("the bus runs F with no want and was not given one: %+v %v", w, ok)
|
||||
}
|
||||
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; ok {
|
||||
t.Errorf("the bus was moved to a new fingerprint with no want: %+v", w)
|
||||
if w, ok := Wants(m, n, nil, nil).Wants["ace"]; !ok || w.FP != fNew {
|
||||
t.Errorf("the bus running an older fingerprint with no want was not given its want by the rules: %+v %v", w, ok)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -34,8 +34,9 @@ func TestManifest(t *testing.T) {
|
||||
reads := slices.Clone(m.Reads)
|
||||
slices.Sort(reads)
|
||||
// ADR 0593 §8: it composes nothing, so it reads no candidate, held or resolved value, compose step or build log; it reads
|
||||
// every node's published compositions to compare them.
|
||||
want := []string{"mesh-delivery-builder.builds", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
|
||||
// every node's published compositions to compare them, and the gate's source and merges, for the release that carries a build (ADR 0601 §1);
|
||||
// and the indexer's module graph, whose verb edges a want waits on (ADR 0614 §2, §3).
|
||||
want := []string{"mesh-delivery-builder.builds", "mesh-delivery-gate.merges", "mesh-delivery-gate.source", "mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers", "mesh-delivery-watchman.presence", "mesh-grants.grants", "mesh-host.composed",
|
||||
"mesh-provisioner.provisions", "node-engine.have"}
|
||||
if !reflect.DeepEqual(reads, want) {
|
||||
|
||||
@@ -25,6 +25,9 @@
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-builder.builds",
|
||||
"mesh-delivery-gate.merges",
|
||||
"mesh-delivery-gate.source",
|
||||
"mesh-delivery-indexer.graph",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-teller.answers",
|
||||
|
||||
@@ -3,7 +3,7 @@ module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -98,7 +98,7 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
|
||||
Description: "Every build that ended since a time, oldest first, from the builder's log build-log (kept 30 days): per build its module, " +
|
||||
"fingerprint, source revision, repository, directory, commit and ref, the build node, when it ended, its outcome (done, failed or " +
|
||||
"queued), the manifest built, the bases it was built against and each artifact made with its digest and reference. more says the " +
|
||||
"log held more than one answer carries: ask again from the last build's time. What mesh-catalog registers a build from. Read-only.",
|
||||
"log held more than one answer carries: ask again from the last build's time. What the operator's command nox builds reads. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"since"}, "properties": map[string]any{
|
||||
"since": map[string]any{"type": "string", "description": "RFC 3339 time; the builds that ended at or after it"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
|
||||
@@ -131,7 +131,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
trunks.start()
|
||||
b.Trunk = trunks.head
|
||||
// The builder's log (ADR 0571 §1): each build's start and end in its module log `build-log`, read by the view, the
|
||||
// indexer and the assigner through their reads, and by mesh-catalog through the tool builder_builds_since.
|
||||
// indexer and the assigner through their reads, and by the operator's command nox builds through the tool builder_builds_since.
|
||||
b.Log = func(e delivery.BuildLogEntry) {
|
||||
if err := appendLog(context.WithoutCancel(ctx), js, cred, e); err != nil {
|
||||
logger.Printf("%v", err)
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -944,7 +944,7 @@ func LogStarted(module string, m delivery.Module, ref, by string, at time.Time)
|
||||
}
|
||||
|
||||
// LogEnded is the log's entry of that build ending: the outcome (done, failed or queued), the manifest built, the bases it
|
||||
// was built against and each artifact made, as the done key has them: what mesh-catalog registers a build from.
|
||||
// was built against and each artifact made, as the done key has them: what the operator's command nox builds shows of a build.
|
||||
func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, result delivery.BuilderBuild) delivery.BuildLogEntry {
|
||||
e := LogStarted(module, m, ref, by, at)
|
||||
e.Event, e.Outcome, e.Manifest = delivery.BuildEndedEvent, result.State, m.Manifest
|
||||
|
||||
@@ -565,6 +565,7 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt
|
||||
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old))
|
||||
r.b.Wait()
|
||||
if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
|
||||
t.Run("a recipe that moves builds the module again under its new fingerprint", aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint)
|
||||
t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls))
|
||||
}
|
||||
// A recipe that differs is a different toolchain too, and a value that names none (written before) is not built.
|
||||
@@ -589,6 +590,45 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0603 §3 (D9): a source revision changing the Go build module's recipe and a Go module builds the module at once with
|
||||
// the recipe the nodes run and again, a second key under the new fingerprint, once the agreement moves to the new recipe;
|
||||
// the first key is neither rebuilt nor waited on.
|
||||
func aRecipeThatMovesBuildsTheModuleAgainUnderItsNewFingerprint(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
r.consider(t, "modules/m")
|
||||
calls := len(r.bm.calls)
|
||||
fp2 := "src2:" + hex64('2')
|
||||
for _, n := range []string{"ace", "novox"} {
|
||||
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "2", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.tools.Put(buildrun.GoBuildModule, n+".go", b, false)
|
||||
}
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if len(r.bm.calls) != calls {
|
||||
t.Errorf("%d calls, was %d: the first fingerprint was built again for the new recipe", len(r.bm.calls), calls)
|
||||
}
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp2,
|
||||
Manifest: mdigest('a'), Path: "modules/m", Toolchains: map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "2"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", data)
|
||||
r.b.Wait()
|
||||
second, err := delivery.BuilderBuildKey("m", fp2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := valueOf(t, r.st, second); v.State != delivery.BuildDone || v.Toolchains["go"].Recipe != "2" {
|
||||
t.Errorf("second build = %+v, want done with recipe 2", v)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
|
||||
t.Errorf("first build = %+v, want it left done", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4
|
||||
r := newRig(t, goBundle)
|
||||
r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false)
|
||||
|
||||
@@ -103,15 +103,25 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{
|
||||
{Name: check.TryTool,
|
||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
|
||||
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"},
|
||||
Description: "Try a delivery group, or one pushed head, before its pull requests are opened (try a group, group try, try several repositories): " +
|
||||
"name the members, each a repository and the exact commit asked (never the branch's head now, so a push after the ask changes nothing of what is tried), " +
|
||||
"or one repository and its head for a try of one member (the seat verb try of mesh-delivery-checker takes members from the registry milestone on; until it is delivered, ask a group through this tool). The checker judges it by the code a group's pull-request check runs, so the two cannot drift: " +
|
||||
"each member merged onto its trunk's head, the unit and the integration level of every member through the build modules, the manifest validator over " +
|
||||
"the members' manifests, the module graph, and the composition on every placed node. Every failure is named (each member, level, test, manifest and node), not " +
|
||||
"the first. A try owes the forge nothing (no status, no Approve asked) and writes no verdict of a pull request; what its passing levels prove is kept, so a pull request's " +
|
||||
"check of the same heads on the same trunk runs only what the try did not pass. A try narrowed to a level or to some tests judges only those, no manifest and no composition, and keeps nothing of tests. " +
|
||||
"Answers the try's id at once; " + check.TryReadTool + " reads where it stands, " + check.TryEndTool + " ends it by hand. A try of the same commits that waits or runs is the answer to a second ask " +
|
||||
"(whatever the order of the members); a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide. A repository the checker checks no level of (one whose trunk the gate does not name) " +
|
||||
"is refused. A try whose ask no build node took within the step bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"members": map[string]any{"type": "array", "description": "the group: one repository at most once, each at the exact commit asked and the trunk commit it is merged onto",
|
||||
"items": map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"}}}},
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>, for a try of one member (instead of members)"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits (with repository)"},
|
||||
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent (with repository)"},
|
||||
"tests": map[string]any{"type": "string", "description": "a go test -run pattern: only those tests run, and nothing is kept"},
|
||||
"level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}, "description": "only this level; both when absent"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
@@ -126,16 +136,32 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
if tr.Now != nil {
|
||||
now = tr.Now
|
||||
}
|
||||
var members []check.TryAskMember
|
||||
if raw, ok := args["members"]; ok {
|
||||
list, isList := raw.([]any)
|
||||
if !isList {
|
||||
return nil, fmt.Errorf("members is a list of {repository, head, trunk}")
|
||||
}
|
||||
for _, item := range list {
|
||||
m, isMap := item.(map[string]any)
|
||||
if !isMap {
|
||||
return nil, fmt.Errorf("a member is {repository, head, trunk}")
|
||||
}
|
||||
members = append(members, check.TryAskMember{Repository: str(m, "repository"), Head: str(m, "head"), Trunk: str(m, "trunk")})
|
||||
}
|
||||
}
|
||||
id, err := check.AskTry(tr.Tries, trunks, check.TryAsk{Repository: str(args, "repository"), Head: str(args, "head"),
|
||||
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level")}, now())
|
||||
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level"), Members: members}, now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"id": id}, nil
|
||||
}},
|
||||
{Name: check.TryReadTool,
|
||||
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
|
||||
Description: "Where a try (of a group or of one head) stands, read whole, as stored: queued, running, or its end (pass, fail, or error when it could not run, or " +
|
||||
"when its manifests and composition could not be judged), with what was said of it, every member's commit and the trunk commit it was merged onto, the time it took, " +
|
||||
"every failure one line each (members, levels, tests, manifests, the composition per node), the levels' tests run and kept, the composition on every placed node, " +
|
||||
"the notes of the check (manifests judged, module graph, kept verdicts); while it runs, each ask it put as soon as it is put, " +
|
||||
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
|
||||
@@ -146,7 +172,7 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
return check.ReadTry(tr.Tries, str(args, "id"))
|
||||
}},
|
||||
{Name: check.TryEndTool,
|
||||
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||
Description: "End a try of a group (or of one head) that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
|
||||
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
|
||||
"takes it within the step bound. A try that ended already is refused.",
|
||||
@@ -170,6 +196,24 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// OpenTriesTools is the tool that lists the tries that wait or run, each with its members at their commits, and whether it is
|
||||
// a whole try (the indexer writes the candidate of a whole try's group): the read a part uses at its start to find the tries
|
||||
// whose start event it missed, instead of reading the checker's state.
|
||||
func OpenTriesTools(tr Trying) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: check.TriesTool,
|
||||
Description: "The tries that wait or run now (a try of a group or of one head; try.started and try.ended on the checker's stream say the same), each with its id, state, its members at their exact commits and " +
|
||||
"trunk commits, and whether it is a whole try (not narrowed to a level or to tests, so it is composed and its manifests judged). For a part that " +
|
||||
"missed the checker's try.started or try.ended event (the indexer at its start); checker_try_read reads one try whole. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
if tr.Tries == nil {
|
||||
return nil, fmt.Errorf("the tries state cannot be read here")
|
||||
}
|
||||
open, err := check.OpenTries(tr.Tries)
|
||||
return map[string]any{"tries": open}, err
|
||||
}}}
|
||||
}
|
||||
|
||||
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
|
||||
const Seat = "mesh-delivery-checker"
|
||||
|
||||
@@ -244,7 +288,9 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). held_by_turn names the candidates whose check an approved " +
|
||||
"merge's turn holds off (novox/hq ADR 0548 §3), with the repository and the merge that waits; the gate's turns tool reads every turn and its " +
|
||||
"hand-now tool steers one. dead_letters says whether the checker hears the asks the build seats give up on (issue 734), and why not: " +
|
||||
"deaf, a try or check whose ask is given up waits until its bound. Read-only.",
|
||||
"deaf, a try or check whose ask is given up waits until its bound. builds_not_ended names, of each running check, the builder's builds of " +
|
||||
"its candidate's changed modules that are neither done nor failed: the check waits on them before it composes, within its build-wait-bound " +
|
||||
"(novox/hq ADR 0603 §10); a failed one fails the check: rebuild it with the builder's builder_build_again, and the check asks itself again when the build is done. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||
}
|
||||
@@ -283,7 +329,7 @@ func main() {
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound(),
|
||||
TurnState: read(stdio.State("mesh-delivery-gate.turns")), HearingState: read(stdio.State(buildrun.HearingState)),
|
||||
AskState: read(stdio.State(check.AsksState))}
|
||||
AskState: read(stdio.State(check.AsksState)), BuildState: read(stdio.State("mesh-delivery-builder.builds"))}
|
||||
r.Runs = runsOf{read(stdio.State("mesh-delivery-indexer.modules"))} // the node-engine at the commit the mesh runs, as the checker reads it
|
||||
tries := stdio.State(check.TriesState)
|
||||
source := stdio.State("mesh-delivery-gate.source")
|
||||
@@ -323,7 +369,7 @@ func main() {
|
||||
_, err := verdicts.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
|
||||
tools := append(append(append(append(Tools(r), TryTools(trying)...), OpenTriesTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
|
||||
if err := stdio.Serve("", logged(tools)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
@@ -106,6 +108,39 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
|
||||
t.Error("the verb tried a repository nothing checks")
|
||||
}
|
||||
// A group try (issue 1921): the members, each an exact commit; members and one repository together are refused; the
|
||||
// description says what it does in the words a search for "try a group" finds.
|
||||
later := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(2, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
g, err := later[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}}})
|
||||
if err != nil || g.(map[string]any)["id"] == id {
|
||||
t.Errorf("a group try = %v, %v, want a try of its own", g, err)
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}},
|
||||
"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40)}); err == nil {
|
||||
t.Error("members and a repository were both taken")
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"members": "novox/mesh-delivery"}); err == nil {
|
||||
t.Error("members that are no list were taken")
|
||||
}
|
||||
for _, tl := range tools {
|
||||
for _, words := range []string{"group"} {
|
||||
if !strings.Contains(strings.ToLower(tl.Description), words) {
|
||||
t.Errorf("%s says nothing of a %s: %q", tl.Name, words, tl.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
if d := strings.ToLower(tools[0].Description); !strings.Contains(d, "try a group") || !strings.Contains(d, "exact commit") || !strings.Contains(d, "every failure") ||
|
||||
!strings.Contains(d, "composition on every placed node") || !strings.Contains(d, "owes the forge nothing") || !strings.Contains(d, "kept") {
|
||||
t.Errorf("the try's description does not say what a group try does: %q", d)
|
||||
}
|
||||
if open := OpenTriesTools(Trying{Tries: store}); len(open) != 1 || open[0].Name != check.TriesTool || !strings.Contains(open[0].Description, "try.started") {
|
||||
t.Errorf("open tries tool = %+v", open)
|
||||
} else if out, err := open[0].Run(nil); err != nil || len(out.(map[string]any)["tries"].([]tryevent.Open)) != 2 {
|
||||
t.Errorf("open tries = %v, %v, want the two tries asked", out, err)
|
||||
}
|
||||
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||
|
||||
@@ -128,7 +128,9 @@ func (m *manifestVerdicts) put(module string, v *delivery.ManifestVerdict) {
|
||||
delete(m.by, module)
|
||||
return
|
||||
}
|
||||
m.by[module] = *v
|
||||
// The sandbox's composer may be the node-engine of an older delivery, which does not know the verdict waits: it is handed a
|
||||
// wait as the refusal it is held as.
|
||||
m.by[module] = v.AsRefused()
|
||||
}
|
||||
|
||||
func (m *manifestVerdicts) all() map[string]delivery.ManifestVerdict {
|
||||
|
||||
@@ -398,7 +398,7 @@ func startBed(t *testing.T) *bed {
|
||||
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
|
||||
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
|
||||
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory, check.TriesBucket: 1,
|
||||
delivery.TurnsBucket: delivery.TurnsHistory} {
|
||||
delivery.TurnsBucket: delivery.TurnsHistory, delivery.BuilderBuildsBucket: 1} {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -578,20 +578,72 @@ func (b *bed) owed(t *testing.T, number int, head, state string) delivery.Owed {
|
||||
// TestCheckerJudgesACandidateAndOwesItsStatus: a candidate the indexer wrote, one pull request of mesh-delivery, is fetched
|
||||
// merged onto the trunk the gate names, digested and tested at both levels on the node, kept, previewed; the verdict passes
|
||||
// with the counts of both levels and the composition not run, and the status mesh/check is owed on the head as success.
|
||||
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1, 1929/AC1
|
||||
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1, 1929/AC1, ADR 0603 §10
|
||||
b := startBed(t)
|
||||
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||
// ADR 0603 §10: the candidate changes the module m, whose manifest the trunk holds; the builder has not built m's
|
||||
// candidate fingerprint yet, so the check waits on the builder's key before it composes.
|
||||
manifest := `{"module":"m"}`
|
||||
digest := delivery.ManifestDigest([]byte(manifest))
|
||||
stored, err := delivery.EncodeManifest(delivery.Manifest{Module: "m", JSON: manifest})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mkey, _ := delivery.ManifestsKey(digest)
|
||||
if _, err := b.buckets[delivery.ManifestsBucket].Put(b.ctx, mkey, stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
known, err := delivery.EncodeModule(delivery.Module{Repository: repo, Commit: trunk, Path: "modules/m", Manifest: digest, None2: "the fingerprint is computed with its chain"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.buckets[delivery.ModulesBucket].Put(b.ctx, "m", known); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fp := "src2:" + strings.Repeat("f", 64)
|
||||
p := pull(repo, 7, head1)
|
||||
p.Modules, p.Fps = []string{"m"}, map[string]delivery.CandidateModule{"m": {FP2: fp, Manifest: digest}}
|
||||
key := b.candidate(t, "feat/x", p)
|
||||
// ADR 0593 §7, §9 (D12): n1 and n2 are placed; n1's engine answers compose-candidate on the bus, n2's never does.
|
||||
asked := b.engine(t, "n1", "n2")
|
||||
b.start(t, nil)
|
||||
v := b.verdict(t, key, ended)
|
||||
if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 2 || v.Levels[delivery.LevelIntegration].Run != 2 || v.Trunk[repo] != trunk {
|
||||
b.said(t, "waits on the builder's builds of m before it composes")
|
||||
if got := asked(); len(got) != 0 {
|
||||
t.Fatalf("the engines were asked %+v before m's build ended", got)
|
||||
}
|
||||
bkey, _ := delivery.BuilderBuildKey("m", fp)
|
||||
building, _ := json.Marshal(delivery.BuilderBuild{V: 1, State: delivery.BuildBuilding})
|
||||
if _, err := b.buckets[delivery.BuilderBuildsBucket].Put(b.ctx, bkey, building); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := b.verdict(t, key, func(delivery.Verdict) bool { return true }); v.State != delivery.VerdictRunning || len(asked()) != 0 {
|
||||
t.Fatalf("verdict %s and %d asks while m builds: the check composes only once its build ended", v.State, len(asked()))
|
||||
}
|
||||
// m's build fails: the check fails naming it. Built again (builder_build_again writes the key done), the check asks
|
||||
// itself again for the same heads, with no recheck, and passes.
|
||||
failedBuild, _ := json.Marshal(delivery.BuilderBuild{V: 1, State: delivery.BuildFailed, Error: "go build: undefined: x"})
|
||||
if _, err := b.buckets[delivery.BuilderBuildsBucket].Put(b.ctx, bkey, failedBuild); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := b.verdict(t, key, ended); v.State != delivery.VerdictFail || v.Failed.Test != check.BuildFailedTest+"m" {
|
||||
t.Fatalf("verdict %+v, want it failed on m's build", v)
|
||||
}
|
||||
built, _ := json.Marshal(delivery.BuilderBuild{V: 1, State: delivery.BuildDone, Manifest: digest})
|
||||
if _, err := b.buckets[delivery.BuilderBuildsBucket].Put(b.ctx, bkey, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.said(t, "the builds of m it failed on are done again, so it is checked again")
|
||||
v := b.verdict(t, key, func(v delivery.Verdict) bool { return v.State == delivery.VerdictPass })
|
||||
// The check asked again takes the packages the failed check ran from their kept verdicts.
|
||||
if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Kept != 2 || v.Levels[delivery.LevelIntegration].Kept != 2 || v.Trunk[repo] != trunk {
|
||||
t.Fatalf("verdict = %+v", v)
|
||||
}
|
||||
if c := v.Composition; c == nil || c.State != delivery.CompositionPass || !slices.Equal(c.Machines, []string{"n1"}) ||
|
||||
!strings.Contains(c.Words, "n2: "+delivery.ComposeNoAnswer) || len(v.Machines) != 1 || v.Machines[0] != "n1" {
|
||||
t.Errorf("composition %+v machines %v", v.Composition, v.Machines)
|
||||
}
|
||||
if v.Composition != nil && strings.Contains(v.Composition.Words, "did not end") {
|
||||
t.Errorf("the composition says %q: m's build ended within the bound", v.Composition.Words)
|
||||
}
|
||||
if got := asked(); len(got) == 0 || got[0].Candidate != key {
|
||||
t.Errorf("n1's engine was asked %+v, want the check's candidate %s", got, key)
|
||||
}
|
||||
@@ -602,10 +654,11 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
|
||||
t.Errorf("the check ask named the nodes %v, want [n1] from the have's node record", names)
|
||||
}
|
||||
o := b.owed(t, 7, head1, "success")
|
||||
if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run") {
|
||||
if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 0 run, 2 kept") {
|
||||
t.Errorf("owed = %+v", o)
|
||||
}
|
||||
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_digest": 2, "mesh-build-go.build_go_test": 2} {
|
||||
// Two checks of the same heads: the one failed on m's build and the one it asked itself again; the second runs no test.
|
||||
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 2, "mesh-build-go.build_go_digest": 4, "mesh-build-go.build_go_test": 2} {
|
||||
if got := b.fake.count(tool); got != want {
|
||||
t.Errorf("%s called %d times, want %d", tool, got, want)
|
||||
}
|
||||
@@ -809,9 +862,11 @@ func (s triesStore) Create(key string, value []byte) error {
|
||||
|
||||
// TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed: a try written as checker_try writes it is taken by the running checker,
|
||||
// judged through the build modules as a pull request's head merged onto its trunk, and ended pass in its key, with no
|
||||
// verdict and no status owed (ADR 0501 §5).
|
||||
// verdict and no status owed (ADR 0501 §5). A whole try waits for the indexer's candidate of its group, which it reads on the
|
||||
// key's watch, and writes the merge previews the indexer makes it from (issue 1921).
|
||||
func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
|
||||
b := startBed(t)
|
||||
asked := b.engine(t, "n1") // a whole try is composed on every placed node, as a group's check is
|
||||
b.start(t, nil)
|
||||
store := triesStore{b.buckets[check.TriesBucket], b.ctx}
|
||||
id, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head1}, time.Now())
|
||||
@@ -819,19 +874,36 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var last check.Try
|
||||
// The indexer's candidate of the try's group (here written by the test), at the heads of its members.
|
||||
raw0, _ := store.Get(id)
|
||||
first, _ := check.DecodeTry(raw0)
|
||||
cand := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: check.TryCase(id, first, nil).HeadsHash,
|
||||
Pulls: []delivery.CandidatePull{{Head: delivery.Head{Repository: repo, Head: head1}, Modules: []string{}}}}
|
||||
cand.FPSet = delivery.FingerprintSet(cand.Pulls)
|
||||
data, err := delivery.EncodeCandidate(cand)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.CandidatesKey(id)
|
||||
if _, err := b.buckets[delivery.CandidatesBucket].Create(b.ctx, ckey, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
raw, _ := store.Get(id)
|
||||
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
|
||||
break
|
||||
}
|
||||
}
|
||||
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" {
|
||||
t.Fatalf("try = %+v, want passed on the trunk", last)
|
||||
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" || last.Composition == nil {
|
||||
t.Fatalf("try = %+v, want passed on the trunk with its composition made", last)
|
||||
}
|
||||
if got := asked(); len(got) != 1 || !strings.HasPrefix(got[0].Candidate, id+".") {
|
||||
t.Errorf("the placed node's engine was asked %+v, want the try's candidate %s.<heads>", got, id)
|
||||
}
|
||||
if b.fake.count("mesh-build-go.build_go_fetch") == 0 {
|
||||
t.Errorf("the try fetched nothing")
|
||||
}
|
||||
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket, delivery.PreviewsBucket} {
|
||||
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket} {
|
||||
if keys, _ := b.buckets[name].Keys(b.ctx); len(keys) != 0 {
|
||||
t.Errorf("a try wrote %s: %v", name, keys)
|
||||
}
|
||||
|
||||
@@ -8,13 +8,17 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
@@ -25,12 +29,19 @@ type judgeDeps struct {
|
||||
trunks *check.Trunks
|
||||
kept check.KV
|
||||
previews check.KV
|
||||
repos *repoModules
|
||||
// candidates answers the indexer's candidate of a try's group (try_candidates.go); nil leaves a whole try with no
|
||||
// manifest or composition judged, said.
|
||||
candidates check.TryCandidates
|
||||
repos *repoModules
|
||||
// published is the indexer's packages state as it is followed (packages.go).
|
||||
published *publishedPackages
|
||||
forge check.Forge
|
||||
announce check.Announcer
|
||||
builds check.BuildStates
|
||||
// ends waits on the builder's builds of a candidate before the check composes (ADR 0603 §10); nil composes without waiting.
|
||||
ends check.BuildEnds
|
||||
// buildWait is the setting build-wait-bound; check.DefaultBuildWait when 0.
|
||||
buildWait time.Duration
|
||||
// compositions makes a check's composition from the engines' answers (ADR 0593 §7); nil leaves it not run.
|
||||
compositions check.CompositionReader
|
||||
// nodes names every node with a record; nil hands no names.
|
||||
@@ -89,6 +100,58 @@ func (n nodeNames) Names(ctx context.Context) ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// runningServes says what the running builds serve (novox/hq ADR 0614 §7), read when a check asks it: every module's have
|
||||
// in the node-engine's have that runs healthy, the builder's build of its fingerprint, and the verbs its manifest serves
|
||||
// (graph.Served). A build whose manifest cannot be read serves nothing here, said in the log.
|
||||
type runningServes struct {
|
||||
have *nodeNames
|
||||
builds check.BuildStates
|
||||
read check.ManifestReader
|
||||
logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
func (r runningServes) Served(ctx context.Context) (func(string) bool, error) {
|
||||
if r.have == nil || r.have.kv == nil || r.builds == nil || r.read == nil {
|
||||
return nil, fmt.Errorf("the node-engine's have, the builder's builds or the indexer's manifests cannot be read")
|
||||
}
|
||||
lister, err := r.have.kv.ListKeys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
haves := map[string]map[string]delivery.Have{}
|
||||
for key := range lister.Keys() {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
e, err := r.have.kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if h, err := delivery.DecodeHave(e.Value()); err == nil {
|
||||
if haves[node] == nil {
|
||||
haves[node] = map[string]delivery.Have{}
|
||||
}
|
||||
haves[node][module] = h
|
||||
}
|
||||
}
|
||||
served, unread := graph.Served(haves, func(module, fp string) ([]string, bool) {
|
||||
b, found, err := r.builds.Build(ctx, module, fp)
|
||||
if err != nil || !found || b.Manifest == "" {
|
||||
return nil, false
|
||||
}
|
||||
m, err := r.read.Manifest(ctx, b.Manifest)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return graph.ServesOf(m), true
|
||||
}, nil)
|
||||
if len(unread) > 0 && r.logf != nil {
|
||||
r.logf("the manifest of the running build of %v cannot be read, so it serves nothing the check counts (ADR 0614 §7)", unread)
|
||||
}
|
||||
return served, nil
|
||||
}
|
||||
|
||||
func newJudge(d judgeDeps) check.Judge {
|
||||
var nodes interface {
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
@@ -96,8 +159,12 @@ func newJudge(d judgeDeps) check.Judge {
|
||||
if d.nodes != nil {
|
||||
nodes = d.nodes
|
||||
}
|
||||
return &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Predefined: d.predefined, Logf: d.logf}
|
||||
ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf}
|
||||
if d.nodes != nil && d.builds != nil && d.repos != nil {
|
||||
ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf}
|
||||
}
|
||||
return ex
|
||||
}
|
||||
|
||||
// manifestKV reads a manifest by its digest.
|
||||
@@ -122,6 +189,42 @@ type repoModules struct {
|
||||
repoOf map[string]string // by module: its repository
|
||||
commit map[string]string // by module: the commit its record was computed at (the commit the mesh runs)
|
||||
typed map[string]string // by module: its manifest digest
|
||||
parsed map[string]catalogue.Manifest // by manifest digest: the manifest as read, once (a digest's manifest never changes)
|
||||
}
|
||||
|
||||
// Manifest is the kept manifest of a digest, read from the indexer's manifests state once and then from memory.
|
||||
func (r *repoModules) Manifest(ctx context.Context, digest string) (catalogue.Manifest, error) {
|
||||
r.mu.Lock()
|
||||
m, known := r.parsed[digest]
|
||||
r.mu.Unlock()
|
||||
if known {
|
||||
return m, nil
|
||||
}
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
raw, _, found, err := r.manifests.Get(ctx, key)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
if !found {
|
||||
return catalogue.Manifest{}, fmt.Errorf("the indexer keeps no manifest %s", digest)
|
||||
}
|
||||
kept, err := delivery.DecodeManifest(raw)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
if m, err = parseManifest(kept.JSON); err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
if r.parsed == nil {
|
||||
r.parsed = map[string]catalogue.Manifest{}
|
||||
}
|
||||
r.parsed[digest] = m
|
||||
r.mu.Unlock()
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Put takes a change of the indexer's modules bucket in: a module's repository and the languages its manifest names.
|
||||
@@ -257,6 +360,69 @@ func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.Bu
|
||||
return b, err == nil, err
|
||||
}
|
||||
|
||||
// buildEnds hears the builder's build keys of a candidate on the bus (mesh-delivery-builder_builds): a watch of exactly
|
||||
// those keys gives each one's value now and every write the builder makes after, so the check waits on the part that owns
|
||||
// the fact and polls nothing (novox/hq ADR 0603 §10).
|
||||
type buildEnds struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (e buildEnds) Ended(ctx context.Context, builds map[string]string) (map[string]delivery.BuilderBuild, error) {
|
||||
return e.until(ctx, builds, func(b delivery.BuilderBuild) bool {
|
||||
return b.State == delivery.BuildDone || b.State == delivery.BuildFailed
|
||||
})
|
||||
}
|
||||
|
||||
func (e buildEnds) Done(ctx context.Context, builds map[string]string) bool {
|
||||
done, err := e.until(ctx, builds, func(b delivery.BuilderBuild) bool { return b.State == delivery.BuildDone })
|
||||
return err == nil && len(done) == len(builds)
|
||||
}
|
||||
|
||||
// until hears the builds' keys until each holds a build that ended holds, or ctx ends, and answers those that do, by module.
|
||||
func (e buildEnds) until(ctx context.Context, builds map[string]string, ended func(delivery.BuilderBuild) bool) (map[string]delivery.BuilderBuild, error) {
|
||||
byKey := map[string]string{}
|
||||
keys := make([]string, 0, len(builds))
|
||||
for module, fp2 := range builds {
|
||||
key, err := delivery.BuilderBuildKey(module, fp2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byKey[key] = module
|
||||
keys = append(keys, key)
|
||||
}
|
||||
out := map[string]delivery.BuilderBuild{}
|
||||
if len(keys) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
w, err := e.kv.WatchFiltered(ctx, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
for len(out) < len(keys) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return out, nil // the bound: the builds not ended are left out
|
||||
case entry, open := <-w.Updates():
|
||||
if !open {
|
||||
return out, nil
|
||||
}
|
||||
if entry == nil || entry.Operation() != jetstream.KeyValuePut {
|
||||
continue // the end of the values now, or a key deleted: it has not ended
|
||||
}
|
||||
module, ours := byKey[entry.Key()]
|
||||
if !ours {
|
||||
continue
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(entry.Value())
|
||||
if err == nil && ended(b) {
|
||||
out[module] = b
|
||||
} else {
|
||||
delete(out, module) // asked again (builder_build_again), or not a build the check can read: not ended
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ManifestDigests is the manifest digest of every module the indexer holds, by module: the catalogue the indexer's judge
|
||||
// judges a change's manifests in (novox/hq ADR 0578 §2).
|
||||
func (r *repoModules) ManifestDigests() map[string]string {
|
||||
|
||||
@@ -170,4 +170,10 @@ func TestAManifestFallingBackToAPredefinedKeyIsRead(t *testing.T) {
|
||||
if _, err := parseManifest(manifest); err != nil {
|
||||
t.Errorf("manifestReader: %v", err)
|
||||
}
|
||||
// The sandbox's composer may be an older node-engine, which knows no verdict waits: it is handed a wait as a refusal.
|
||||
var held manifestVerdicts
|
||||
held.put("nats", &delivery.ManifestVerdict{Module: "nats", State: delivery.VerdictManifestWaits, Unknown: []string{"core-credentials"}, Judge: "j"})
|
||||
if v := held.all()["nats"]; v.State != delivery.VerdictRefused || len(v.Problems) != 1 || len(v.Unknown) != 0 {
|
||||
t.Errorf("a waiting verdict is handed to the sandbox as %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -166,14 +167,19 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
|
||||
// says nothing of the builds, until the checker starts again.
|
||||
var builds check.BuildStates
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds until the checker starts again", logger.Printf); kv != nil {
|
||||
builds = buildReader{kvStore{kv}}
|
||||
var ends check.BuildEnds
|
||||
if kv := process.OptionalBucket(ctx, js, delivery.BuilderBuildsBucket, "the line says nothing of the builds, and a check composes without waiting on them, until the checker starts again", logger.Printf); kv != nil {
|
||||
builds, ends = buildReader{kvStore{kv}}, buildEnds{kv}
|
||||
}
|
||||
if ends != nil {
|
||||
ch.Rebuilds = ends // a check failed on a build is asked again once that build is done again (ADR 0603 §10)
|
||||
}
|
||||
// The check's composition is the checker's, from what each placed node's engine answers (ADR 0593 §7, §9).
|
||||
answerBound, err := composeAnswerBound(getenv("MESH_CHECKER_SETTINGS"))
|
||||
set, err := readSettings(getenv("MESH_CHECKER_SETTINGS"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
answerBound := set.answer
|
||||
holdCallsFor(answerBound)
|
||||
verdicts := &manifestVerdicts{}
|
||||
compositions := check.EngineCompositions{
|
||||
@@ -184,7 +190,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Verdicts: verdicts.all,
|
||||
}
|
||||
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf,
|
||||
// A whole try's start and end are said on the stream checks, for the indexer to write and delete the candidate of its group.
|
||||
Say: func(ctx context.Context, event string, body tryevent.Event) error {
|
||||
data, _ := json.Marshal(body)
|
||||
_, err := js.Publish(ctx, eventSubject(event), data, jetstream.WithRetryAttempts(10), jetstream.WithRetryWait(time.Second))
|
||||
return err
|
||||
}}
|
||||
announce := func(ctx context.Context, event, key string) {
|
||||
body, _ := json.Marshal(map[string]string{"key": key})
|
||||
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
||||
@@ -197,6 +209,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The indexer's packages state (ADR 0593 §2): followed once the bus has it, so a Go check builds every mesh package at
|
||||
// its published version; until then a check builds each at the version its go.mod names.
|
||||
published := &publishedPackages{}
|
||||
compositions.Published = published
|
||||
process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "a Go check builds each mesh package at the version its go.mod names",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
w, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
@@ -211,8 +224,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
|
||||
ch.Judge = newJudge(judgeDeps{
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
||||
repos: repos, published: published, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, candidates: tryCandidates{buckets[delivery.CandidatesBucket]},
|
||||
repos: repos, published: published, builds: builds, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||
announce: announce, compositions: compositions,
|
||||
nodes: openNodeNames(ctx, js, logger.Printf),
|
||||
predefined: keptPredefined{process.AwaitBucket(ctx, js, conn, delivery.IndexerModule+"_"+delivery.IndexerPredefinedState,
|
||||
@@ -328,6 +341,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
})
|
||||
}()
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if check.IsTryGroup(e.Key()) {
|
||||
return // a group try's candidate: its try judges it, and the forge is owed nothing (issue 1921)
|
||||
}
|
||||
if opOf(e) {
|
||||
ch.Candidate(ctx, e.Key(), e.Value())
|
||||
} else {
|
||||
|
||||
@@ -10,38 +10,57 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// The checker's own settings (its config, written by the node-engine into ${dir:state}/settings.json, named to the
|
||||
// process by MESH_CHECKER_SETTINGS): compose-answer-bound, the seconds a check waits for a node's engine to answer
|
||||
// compose-candidate (novox/hq ADR 0593 §7, §9: 60 by default; the checker owns it).
|
||||
// compose-candidate (novox/hq ADR 0593 §7, §9: 60 by default), and build-wait-bound, the seconds a check waits for the
|
||||
// builder's builds of its candidate before it composes (ADR 0603 §10: 1800 by default). The checker owns both.
|
||||
|
||||
// composeAnswerBound reads compose-answer-bound from the settings file at path: delivery.ComposeAnswerBound where no
|
||||
// file or no value is there; a value that is no number of seconds from 1 to 600 is refused, naming it.
|
||||
func composeAnswerBound(path string) (time.Duration, error) {
|
||||
// settings are the checker's settings as read.
|
||||
type settings struct {
|
||||
// answer is compose-answer-bound; buildWait is build-wait-bound.
|
||||
answer, buildWait time.Duration
|
||||
}
|
||||
|
||||
// readSettings reads the settings file at path: each value's default where no file or no value is there; a value that is
|
||||
// no whole number of seconds within its range is refused, naming it.
|
||||
func readSettings(path string) (settings, error) {
|
||||
out := settings{answer: delivery.ComposeAnswerBound, buildWait: check.DefaultBuildWait}
|
||||
if path == "" {
|
||||
return delivery.ComposeAnswerBound, nil
|
||||
return out, nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return delivery.ComposeAnswerBound, nil
|
||||
return out, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return settings{}, err
|
||||
}
|
||||
var s struct {
|
||||
Bound *float64 `json:"compose-answer-bound"`
|
||||
Answer *float64 `json:"compose-answer-bound"`
|
||||
BuildWait *float64 `json:"build-wait-bound"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return 0, fmt.Errorf("%s is no settings file: %w", path, err)
|
||||
return settings{}, fmt.Errorf("%s is no settings file: %w", path, err)
|
||||
}
|
||||
if s.Bound == nil {
|
||||
return delivery.ComposeAnswerBound, nil
|
||||
for _, v := range []struct {
|
||||
name string
|
||||
value *float64
|
||||
min, max float64
|
||||
into *time.Duration
|
||||
}{{"compose-answer-bound", s.Answer, 1, 600, &out.answer}, {"build-wait-bound", s.BuildWait, 60, 7200, &out.buildWait}} {
|
||||
if v.value == nil {
|
||||
continue
|
||||
}
|
||||
if *v.value < v.min || *v.value > v.max || *v.value != float64(int(*v.value)) {
|
||||
return settings{}, fmt.Errorf("%s is %v: a whole number of seconds from %v to %v", v.name, *v.value, v.min, v.max)
|
||||
}
|
||||
*v.into = time.Duration(*v.value) * time.Second
|
||||
}
|
||||
if *s.Bound < 1 || *s.Bound > 600 || *s.Bound != float64(int(*s.Bound)) {
|
||||
return 0, fmt.Errorf("compose-answer-bound is %v: a whole number of seconds from 1 to 600", *s.Bound)
|
||||
}
|
||||
return time.Duration(*s.Bound) * time.Second, nil
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// holdCallsFor makes a bus call wait the compose-answer-bound (novox/hq ADR 0593 §7), not the SDK's CallTimeout of 30 s:
|
||||
|
||||
@@ -8,19 +8,25 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// ADR 0593 §7: the answer bound is the checker's setting compose-answer-bound, 60 s when the file or the value is
|
||||
// absent; a value that is no whole number of seconds from 1 to 600 stops the start, naming it.
|
||||
// absent; a value that is no whole number of seconds from 1 to 600 stops the start, naming it. ADR 0603 §10: the wait for
|
||||
// the candidate's builds is build-wait-bound, 30 minutes when absent, from 60 to 7200 seconds.
|
||||
func TestTheAnswerBoundIsTheCheckersSetting(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for _, row := range []struct {
|
||||
body string
|
||||
want time.Duration
|
||||
err bool
|
||||
}{{"", delivery.ComposeAnswerBound, false}, {`{}`, delivery.ComposeAnswerBound, false},
|
||||
{`{"compose-answer-bound": 90}`, 90 * time.Second, false}, {`{"compose-answer-bound": 0}`, 0, true},
|
||||
{`{"compose-answer-bound": 1.5}`, 0, true}} {
|
||||
body string
|
||||
want time.Duration
|
||||
err bool
|
||||
buildWait time.Duration
|
||||
}{{"", delivery.ComposeAnswerBound, false, check.DefaultBuildWait}, {`{}`, delivery.ComposeAnswerBound, false, check.DefaultBuildWait},
|
||||
{`{"compose-answer-bound": 90}`, 90 * time.Second, false, check.DefaultBuildWait}, {`{"compose-answer-bound": 0}`, 0, true, 0},
|
||||
{`{"compose-answer-bound": 1.5}`, 0, true, 0},
|
||||
{`{"compose-answer-bound": 60, "build-wait-bound": 600}`, 60 * time.Second, false, 10 * time.Minute},
|
||||
{`{"build-wait-bound": 30}`, 0, true, 0}} {
|
||||
path := filepath.Join(dir, "settings.json")
|
||||
os.Remove(path)
|
||||
if row.body != "" {
|
||||
@@ -28,9 +34,10 @@ func TestTheAnswerBoundIsTheCheckersSetting(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := composeAnswerBound(path)
|
||||
if (err != nil) != row.err || got != row.want {
|
||||
t.Errorf("%q: %s, %v", row.body, got, err)
|
||||
set, err := readSettings(path)
|
||||
got := set.answer
|
||||
if (err != nil) != row.err || got != row.want || set.buildWait != row.buildWait {
|
||||
t.Errorf("%q: %s, %s, %v", row.body, got, set.buildWait, err)
|
||||
}
|
||||
// Issue 1126: the SDK's call gives up at CallTimeout (30 s), so the checker's calls must wait the bound read
|
||||
// (an answer at 31 s is heard under 60 s), and the bound never shortens the SDK's own.
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// candidateWaitsOnPreview is the None2 of a module the indexer wrote from a pull message alone, before it had the merge
|
||||
// preview of the head (the indexer's NoneNoPreview): its fingerprint comes with the preview, so a candidate that still says
|
||||
// it is not yet the candidate of the heads previewed. It moves to mesh-sdk's delivery package with the next change there.
|
||||
const candidateWaitsOnPreview = "no merge preview is computed"
|
||||
|
||||
// tryCandidates answers the indexer's candidate of a try's group (novox/hq ADR 0593 §7, issue 1921): the candidate the
|
||||
// engines compose and the manifest validator judges, which the indexer makes of the try's previews. It waits for it on the
|
||||
// key's own watch, with no timer between reads, and gives up at the bound.
|
||||
type tryCandidates struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (t tryCandidates) TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error) {
|
||||
key, err := delivery.CandidatesKey(group)
|
||||
if err != nil {
|
||||
return delivery.Candidate{}, err
|
||||
}
|
||||
if bound <= 0 {
|
||||
bound = time.Minute
|
||||
}
|
||||
waiting, cancel := context.WithTimeout(ctx, bound)
|
||||
defer cancel()
|
||||
w, err := t.kv.Watch(waiting, key) // the value now, then each change
|
||||
if err != nil {
|
||||
return delivery.Candidate{}, fmt.Errorf("the indexer's candidate %s cannot be watched: %w", key, err)
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
last := "the indexer holds none"
|
||||
for {
|
||||
select {
|
||||
case <-waiting.Done():
|
||||
return delivery.Candidate{}, fmt.Errorf("the indexer wrote no candidate of %s at heads %s within %s (%s)", group, heads, bound, last)
|
||||
case e := <-w.Updates():
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(e.Value())
|
||||
switch {
|
||||
case err != nil:
|
||||
last = "what it holds is not a candidate: " + err.Error()
|
||||
case c.HeadsHash != heads:
|
||||
last = "it holds heads " + c.HeadsHash
|
||||
case waitsOnPreview(c):
|
||||
last = "it has not yet the merge previews' fingerprints"
|
||||
default:
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitsOnPreview says a module of the candidate has no fingerprint yet because its preview was not read.
|
||||
func waitsOnPreview(c delivery.Candidate) bool {
|
||||
for _, p := range c.Pulls {
|
||||
for _, m := range p.Fps {
|
||||
if m.None2 == candidateWaitsOnPreview {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
{
|
||||
"compose-answer-bound": 60
|
||||
"compose-answer-bound": 60,
|
||||
"build-wait-bound": 1800
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
@@ -25,3 +25,7 @@ require (
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -29,24 +29,30 @@ const (
|
||||
nodeEngineModule = "mesh-host"
|
||||
// networkingRepository holds route-proxy, whose manifest the node-engine's proofs of a composition read.
|
||||
networkingRepository = "novox/mesh-networking"
|
||||
// contractsRepository and deliveryRepository hold the seat definitions mesh-sdk's bootstrapped seats are compared with: its
|
||||
// tests read both beside it (beside.Dir "mesh-contracts" and "mesh-delivery"), at the group's head for a member, else at the
|
||||
// trunk (a group try lays them out the same way: besideFor serves the check and the try).
|
||||
contractsRepository = "novox/mesh-contracts"
|
||||
deliveryRepository = DeliveryRepository
|
||||
)
|
||||
|
||||
// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside
|
||||
// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog
|
||||
// checks/deps). Which commit of each is besideCommit's rule. No repository's tests read the controller, which ended by its
|
||||
// unassignment (novox/hq ADR 0590, issue 1126), and its repository is emptied and archived (ADR 0593 §11), so it has no
|
||||
// siblings placed either. The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
|
||||
// siblings placed either. mesh-sdk reads mesh-contracts and mesh-delivery (its bootstrapped seats against their contracts files). The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
|
||||
// §7): mesh-build-go builds it for a compose ask. The node-engine's read mesh-networking since route-proxy's proofs of a
|
||||
// composition moved into them (ADR 0593).
|
||||
var besideRules = map[string][]string{
|
||||
nodeEngineRepo: {catalogueRepository, SDKRepository, networkingRepository},
|
||||
SDKRepository: {catalogueRepository, nodeEngineRepo},
|
||||
SDKRepository: {catalogueRepository, nodeEngineRepo, contractsRepository, deliveryRepository},
|
||||
catalogueRepository: {SDKRepository, nodeEngineRepo},
|
||||
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
|
||||
}
|
||||
|
||||
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the node-engine at the commit the mesh runs
|
||||
// (the commit of the indexer's module mesh-host, its trunk head when none is known); any other sibling, the catalogue
|
||||
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the node-engine at the head of the group's pull
|
||||
// request for it when the group holds one, else at the commit the mesh runs (the commit of the indexer's module mesh-host,
|
||||
// its trunk head when none is known); any other sibling, the catalogue
|
||||
// included, at the head of the group's pull request for it when the group holds one, else at its trunk, since a group's
|
||||
// check judges each member against the group's heads (ADR 0482 §2, ADR 0573 §4; issue 1126: a judge a group changed was
|
||||
// judged against the catalogue's trunk). And first, the change's own repository at the trunk it is merged onto, so a test
|
||||
@@ -63,6 +69,12 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
}
|
||||
for _, repo := range besideRules[p.Repository] {
|
||||
switch {
|
||||
case repo == nodeEngineRepo && c.member(repo) != nil:
|
||||
// A group's check judges each member against the group's heads (ADR 0573 §4, ADR 0482 §2), and the group's
|
||||
// node-engine is what runs after its merge: it is placed at the member's head, as any member is. Neither atTrunk
|
||||
// nor runs, so no move of the trunk or of the running commit puts the verdict out of date (besideAtTrunk).
|
||||
member := c.member(repo)
|
||||
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
|
||||
case repo == nodeEngineRepo:
|
||||
commit := ""
|
||||
if j.Runs != nil {
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
)
|
||||
|
||||
// What the check builds of a candidate before it composes (novox/hq ADR 0603 §10). The builder builds every fingerprint the
|
||||
// indexer's candidate carries, as a key of its own (builds.<module>.<fingerprint>), with the build modules that run; the
|
||||
// check adds no build path of its own and waits on those keys (awaitBuilds), so the node-engines compose the candidate's
|
||||
// modules with their builds. It does not wait on two kinds of changed module, and names them in its composition's warnings:
|
||||
//
|
||||
// - one built by a build module the change alters (the recipe of mesh-build-go or mesh-build-node enters the fingerprint
|
||||
// of every module it builds, ADR 0530 §1): its candidate build is the running build module's, and the check cannot run
|
||||
// the candidate's build module to build it (it runs it only for its own digest and tests, ADR 0535 §1), so it is not
|
||||
// composed with a build of the changed build module; it is built again once the agreed toolchain moves (ADR 0603 §3).
|
||||
// The build module's own build is waited on, as any other: the running build module builds its new source.
|
||||
// - one that stands on a module the change alters (build.on): its candidate fingerprint names the base's trunk
|
||||
// fingerprint, and no build module builds on another module's artifact (ADR 0487 §1), so its build stands on the
|
||||
// trunk's base; the base's own build is waited on.
|
||||
//
|
||||
// A module the trunk holds no record of yet is not built by the builder until it is merged, so it is not waited on either.
|
||||
//
|
||||
// A changed build module whose manifest invokes a seat verb or module tool no running build serves is one the check cannot
|
||||
// run (ADR 0614 §7): the modules it builds are composed from their manifests, as above, and the words name the verb.
|
||||
|
||||
// buildsOfTheCheck are the builds the check waits on before it composes (module to fingerprint) and the words for the
|
||||
// changed modules it composes without waiting on their build, one line each, sorted. A manifest that cannot be read leaves
|
||||
// every build waited on and nothing said of build modules or bases (nothing is said that was not read).
|
||||
func (st *state) buildsOfTheCheck(ctx context.Context) (map[string]string, []string) {
|
||||
wait := map[string]string{}
|
||||
changed := map[string]catalogue.Manifest{}
|
||||
readable := true
|
||||
for _, p := range st.c.Pulls {
|
||||
for module, cm := range p.Fps {
|
||||
if cm.FP2 == "" || cm.Manifest == "" {
|
||||
continue // the builder builds no fingerprint of it (candidates.go: none of the new form, or no manifest)
|
||||
}
|
||||
wait[module] = cm.FP2
|
||||
if !readable || st.j.Read == nil {
|
||||
readable = false
|
||||
continue
|
||||
}
|
||||
m, err := st.j.Read.Manifest(ctx, cm.Manifest)
|
||||
if err != nil {
|
||||
st.j.logf("%s: the manifest of %s cannot be read, so the modules built with a changed build module or base are not named and every build is waited on: %v", st.c.Key, module, err)
|
||||
readable = false
|
||||
continue
|
||||
}
|
||||
changed[module] = m
|
||||
}
|
||||
}
|
||||
var words []string
|
||||
if st.j.Runs != nil {
|
||||
var unheld []string
|
||||
for _, module := range slices.Sorted(maps.Keys(wait)) {
|
||||
if st.j.Runs.CommitOf(module) == "" {
|
||||
unheld = append(unheld, module)
|
||||
delete(wait, module)
|
||||
}
|
||||
}
|
||||
if len(unheld) > 0 {
|
||||
words = append(words, fmt.Sprintf("%s: the trunk holds no record of it yet, so the builder builds it once it is merged and the check composes it without a build (ADR 0603 §10)",
|
||||
strings.Join(unheld, ", ")))
|
||||
}
|
||||
}
|
||||
if !readable {
|
||||
return wait, words
|
||||
}
|
||||
for _, bm := range slices.Sorted(maps.Keys(buildrun.LanguageOf)) {
|
||||
if _, alters := changed[bm]; !alters {
|
||||
continue
|
||||
}
|
||||
var built []string
|
||||
for module, m := range changed {
|
||||
if _, isBuildModule := buildrun.LanguageOf[module]; !isBuildModule && buildsWith(m, buildrun.LanguageOf[bm]) {
|
||||
built = append(built, module)
|
||||
delete(wait, module)
|
||||
}
|
||||
}
|
||||
slices.Sort(built)
|
||||
if len(built) > 0 {
|
||||
words = append(words, fmt.Sprintf("the change alters the build module %s: %s, which it builds, are built by the running %s, not the changed one, which the check cannot run, so the check does not wait on their builds; they are built again once the agreed toolchain moves (ADR 0603 §3, §10)",
|
||||
bm, strings.Join(built, ", "), bm))
|
||||
}
|
||||
if unserved := st.unservedVerbs(ctx, changed[bm]); len(unserved) > 0 {
|
||||
what := "nothing it builds is in the change"
|
||||
if len(built) > 0 {
|
||||
what = strings.Join(built, ", ") + ", which it builds, are composed from their manifests"
|
||||
}
|
||||
words = append(words, fmt.Sprintf("the changed build module %s invokes %s, which no running build serves, so the check cannot run it: %s (ADR 0614 §7)",
|
||||
bm, strings.Join(unserved, ", "), what))
|
||||
}
|
||||
}
|
||||
g := graph.OfManifests(changed)
|
||||
for _, base := range slices.Sorted(maps.Keys(changed)) {
|
||||
if on := g.Dependents(base); len(on) > 0 {
|
||||
for _, module := range on {
|
||||
delete(wait, module)
|
||||
}
|
||||
words = append(words, fmt.Sprintf("the change alters %s, a base of %s: their builds stand on the trunk's %s, since no build module builds on a candidate's base, so the check waits on the build of %s alone (ADR 0603 §10)",
|
||||
base, strings.Join(on, ", "), base, base))
|
||||
}
|
||||
}
|
||||
return wait, words
|
||||
}
|
||||
|
||||
// buildsWith says whether a manifest has a bundle in the language: the artifacts whose fingerprint names that language's
|
||||
// toolchain and its build module's recipe (an archive is packed with no toolchain, ADR 0478 clause 1).
|
||||
func buildsWith(m catalogue.Manifest, language string) bool {
|
||||
return m.Build != nil && slices.ContainsFunc(m.Build.Artifacts, func(a catalogue.Artifact) bool {
|
||||
return a.Kind == catalogue.ArtifactBundle && a.Language == language
|
||||
})
|
||||
}
|
||||
|
||||
// unservedVerbs are the seat verbs and module tools m invokes (graph.InvokesOf) that no running build serves, sorted; none
|
||||
// when nothing reads what runs, or what runs cannot be read (said in the log: nothing is said that was not read).
|
||||
func (st *state) unservedVerbs(ctx context.Context, m catalogue.Manifest) []string {
|
||||
if st.j.Serving == nil {
|
||||
return nil
|
||||
}
|
||||
served, err := st.j.Serving.Served(ctx)
|
||||
if err != nil {
|
||||
st.j.logf("%s: what the running builds serve cannot be read, so no build module is named for a verb it invokes: %v", st.c.Key, err)
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, v := range graph.InvokesOf(m) {
|
||||
if !served(v) {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -2,7 +2,12 @@ package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -64,3 +69,102 @@ func (j *Examiner) buildNotes(ctx context.Context, c *Case) []BuildNote {
|
||||
}
|
||||
return notes
|
||||
}
|
||||
|
||||
// DefaultBuildWait is how long a check waits for the candidate's builds to end before it composes (the setting
|
||||
// build-wait-bound, novox/hq ADR 0603 §10): past it the check composes without the builds not ended and names them.
|
||||
const DefaultBuildWait = 30 * time.Minute
|
||||
|
||||
// BuildEnds waits on the builder's build keys (builds.<module>.<fingerprint>), hearing each key's changes on the bus as the
|
||||
// builder writes them: the builder owns the fact, and nothing is polled.
|
||||
type BuildEnds interface {
|
||||
// Ended waits until the build of every module at its fingerprint (module to fingerprint) is done or failed, or ctx ends,
|
||||
// and answers the builds that ended, by module. An error is only builds that cannot be heard at all.
|
||||
Ended(ctx context.Context, builds map[string]string) (map[string]delivery.BuilderBuild, error)
|
||||
// Done waits until the build of every module at its fingerprint is done (a failed one built again, builder_build_again),
|
||||
// and says whether they all are; false when ctx ends first or they cannot be heard.
|
||||
Done(ctx context.Context, builds map[string]string) bool
|
||||
}
|
||||
|
||||
// BuildFailedTest is what a failure of the check names a failed build of the candidate by: the test `build: <module>`.
|
||||
// A verdict failed on builds is asked again by the checker itself once each is built again (Checker.Rebuilt).
|
||||
const BuildFailedTest = "build: "
|
||||
|
||||
// failedBuilds are the candidate's builds a failed verdict names as failed (BuildFailedTest), by module, at the fingerprint
|
||||
// the case carries for it; none for a verdict that names no failed build.
|
||||
func failedBuilds(v delivery.Verdict, cs *Case) map[string]string {
|
||||
if v.State != delivery.VerdictFail {
|
||||
return nil
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, f := range append([]delivery.FailedFirst{v.Failed}, v.Failures...) {
|
||||
module, named := strings.CutPrefix(f.Test, BuildFailedTest)
|
||||
if !named {
|
||||
continue
|
||||
}
|
||||
for _, p := range cs.Pulls {
|
||||
if cm, ok := p.Fps[module]; ok && cm.FP2 != "" {
|
||||
out[module] = cm.FP2
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (j *Examiner) buildWait() time.Duration {
|
||||
if j.BuildWait > 0 {
|
||||
return j.BuildWait
|
||||
}
|
||||
return DefaultBuildWait
|
||||
}
|
||||
|
||||
// awaitBuilds waits for the candidate's builds before the check composes (ADR 0603 §10): the check builds what the builder
|
||||
// builds of the candidate and composes with it. A failed build fails the check, naming the module and the builder's words;
|
||||
// a build not ended within the bound is named, and the check composes without it. It answers the words of the composition's
|
||||
// warnings: what the check composes without waiting on its build, and why.
|
||||
func (st *state) awaitBuilds(ctx context.Context, v *delivery.Verdict) []string {
|
||||
wait, words := st.buildsOfTheCheck(ctx)
|
||||
if len(wait) == 0 {
|
||||
return words
|
||||
}
|
||||
modules := slices.Sorted(maps.Keys(wait))
|
||||
if v.State == delivery.VerdictFail {
|
||||
// The check has failed already: its red status is never held back by a build (the lead, 2026-11-03).
|
||||
return append(words, fmt.Sprintf("the check failed before its builds were waited on, so it composes at once, without waiting on the builds of %s (ADR 0603 §10)", strings.Join(modules, ", ")))
|
||||
}
|
||||
if st.j.BuildEnds == nil {
|
||||
return append(words, fmt.Sprintf("the builds of %s cannot be heard, so the check composes without waiting on them (ADR 0603 §10)", strings.Join(modules, ", ")))
|
||||
}
|
||||
bound := st.j.buildWait()
|
||||
waits := fmt.Sprintf("waits on the builder's builds of %s before it composes, at most %s (ADR 0603 §10)", strings.Join(modules, ", "), bound)
|
||||
st.c.note(waits)
|
||||
st.j.logf("%s %s", st.c.Key, waits)
|
||||
at, cancel := context.WithTimeout(ctx, bound)
|
||||
ended, err := st.j.BuildEnds.Ended(at, wait)
|
||||
cancel()
|
||||
if err != nil {
|
||||
st.j.logf("%s: the builds of %s cannot be heard: %v", st.c.Key, strings.Join(modules, ", "), err)
|
||||
return append(words, fmt.Sprintf("the builds of %s cannot be heard (%v), so the check composes without waiting on them (ADR 0603 §10)", strings.Join(modules, ", "), err))
|
||||
}
|
||||
var late []string
|
||||
for _, module := range modules {
|
||||
b, done := ended[module]
|
||||
switch {
|
||||
case !done:
|
||||
late = append(late, module)
|
||||
case b.State == delivery.BuildFailed:
|
||||
f := delivery.FailedFirst{Test: BuildFailedTest + module, Words: fmt.Sprintf("the candidate's build at %s failed (rebuild with builder_build_again; the check asks itself again when the build is done): %s", wait[module], b.Error)}
|
||||
if v.State != delivery.VerdictFail {
|
||||
v.State, v.Failed = delivery.VerdictFail, f
|
||||
continue
|
||||
}
|
||||
addFailure(v, f)
|
||||
}
|
||||
}
|
||||
if len(late) > 0 && ctx.Err() == nil {
|
||||
words = append(words, fmt.Sprintf("the builds of %s did not end within the build-wait-bound of %s, so the check composes without them (ADR 0603 §10)", strings.Join(late, ", "), bound))
|
||||
}
|
||||
return words
|
||||
}
|
||||
|
||||
@@ -55,6 +55,8 @@ type Case struct {
|
||||
Watch func(askID string, step *delivery.Step)
|
||||
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
||||
Progress *Progress
|
||||
// Wait says what a whole try waits for now ("" when nothing); the try keeps it for try-read. Nil for a pull request's check.
|
||||
Wait func(why string)
|
||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
|
||||
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
|
||||
Kind string
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -79,6 +81,9 @@ type Checker struct {
|
||||
}
|
||||
// Bound is how long one check may run; none when 0 (DefaultBound).
|
||||
Bound time.Duration
|
||||
// Rebuilds hears the builder's build keys a failed verdict names (failedBuilds): once each is written done, the check of
|
||||
// those heads is asked again by the checker itself (ADR 0603 §10), as a failure of the nodes is; nil asks none again.
|
||||
Rebuilds BuildEnds
|
||||
// Asks keeps the asks each running check put, by verdict key (AsksState), for checker_ask_again; nil keeps none.
|
||||
Asks AsksKV
|
||||
// Link is the base of the address a status points at; the verdict key follows.
|
||||
@@ -104,6 +109,8 @@ type Checker struct {
|
||||
held map[string]string // by verdict key: the turn that holds its check off, as last said
|
||||
live map[string]liveCheck // by verdict key: the check running in this process, ended when its heads go
|
||||
settling map[string]*window // by candidates key: the window its heads settle in
|
||||
rebuilt map[string]bool // by verdict key: every failed build its verdict names is done again
|
||||
awaiting map[string]bool // by verdict key: its failed builds are heard (awaitRebuilt)
|
||||
// outsideSaid are the candidates outside the mesh said in the log, by group and heads.
|
||||
outsideSaid map[string]bool
|
||||
}
|
||||
@@ -162,6 +169,7 @@ func (ch *Checker) init() {
|
||||
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||
ch.nodesFail, ch.held = map[string]uint64{}, map[string]string{}
|
||||
ch.live, ch.settling = map[string]liveCheck{}, map[string]*window{}
|
||||
ch.rebuilt, ch.awaiting = map[string]bool{}, map[string]bool{}
|
||||
ch.outsideSaid = map[string]bool{}
|
||||
})
|
||||
}
|
||||
@@ -380,8 +388,9 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
||||
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
|
||||
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
|
||||
// ADR 0536 §4). The checker holds no place and no capacity of its own.
|
||||
ch.prepare(ctx, cs)
|
||||
ch.prepare(ctx, key, cs)
|
||||
ch.try(ctx, run, key, cs)
|
||||
ch.hearFailedBuilds(ctx, key, cs) // a check it failed on a build is asked again once that build is done again
|
||||
ch.mu.Lock()
|
||||
more := ch.again[vkey]
|
||||
delete(ch.again, vkey)
|
||||
@@ -518,7 +527,7 @@ const (
|
||||
// or sibling commits that moved since is carried, its packages run only where the move changed their digest, and a pass's
|
||||
// commits are named as the carry's CarriedFrom. A verdict that stands is ranked with the carries: its claim finds nothing
|
||||
// to do.
|
||||
func (ch *Checker) prepare(ctx context.Context, cs *Case) {
|
||||
func (ch *Checker) prepare(ctx context.Context, group string, cs *Case) {
|
||||
cs.Kind, cs.CarriedFrom, cs.Composition = KindNew, nil, nil
|
||||
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
||||
if err != nil || !found {
|
||||
@@ -531,6 +540,10 @@ func (ch *Checker) prepare(ctx context.Context, cs *Case) {
|
||||
case IsNodesFailure(v):
|
||||
cs.Kind = KindNodes
|
||||
return
|
||||
case failedBuilds(v, cs) != nil:
|
||||
cs.Kind = KindNodes // asked again on check once its builds are done again (awaitRebuilt), never by a recheck
|
||||
ch.awaitRebuilt(ctx, group, cs.Key, failedBuilds(v, cs))
|
||||
return
|
||||
case IsConflictFailure(v):
|
||||
return // a head git refused: its next check is whole (ADR 0548 §7)
|
||||
case IsEnded(v), IsUntrunked(v):
|
||||
@@ -687,6 +700,8 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
||||
return nil, nil
|
||||
case IsEnded(cur), ch.stale(cur, cs):
|
||||
retake = true // heads held again after their check was ended unjudged (issue 1890), or a trunk that moved
|
||||
case cur.State == delivery.VerdictFail && ch.takeRebuilt(cs.Key):
|
||||
retake = true // every failed build it names is done again (ADR 0603 §10)
|
||||
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
|
||||
ch.mu.Lock()
|
||||
at, ok := ch.nodesFail[cs.Key]
|
||||
@@ -1070,3 +1085,61 @@ func (ch *Checker) movedSince(cs *Case) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// awaitRebuilt hears, once per verdict key, the builder's keys of the builds a failed verdict names: when each is written
|
||||
// done again (builder_build_again), the check of those heads is asked again (claim retakes it), as a failure of the nodes
|
||||
// is after the nodes changed, so no recheck by hand is needed (ADR 0603 §10). Heard on the builder's own writes, never
|
||||
// polled; it ends with the checker.
|
||||
func (ch *Checker) awaitRebuilt(ctx context.Context, group, vkey string, builds map[string]string) {
|
||||
if ch.Rebuilds == nil {
|
||||
return
|
||||
}
|
||||
ch.mu.Lock()
|
||||
if ch.awaiting[vkey] {
|
||||
ch.mu.Unlock()
|
||||
return
|
||||
}
|
||||
ch.awaiting[vkey] = true
|
||||
ch.mu.Unlock()
|
||||
ch.wg.Add(1)
|
||||
go func() {
|
||||
defer ch.wg.Done()
|
||||
done := ch.Rebuilds.Done(ctx, builds)
|
||||
ch.mu.Lock()
|
||||
delete(ch.awaiting, vkey)
|
||||
if done {
|
||||
ch.rebuilt[vkey] = true
|
||||
}
|
||||
c, known := ch.known[group]
|
||||
ch.mu.Unlock()
|
||||
if done && known && ctx.Err() == nil {
|
||||
ch.logf("%s: the builds of %s it failed on are done again, so it is checked again (ADR 0603 §10)", vkey, strings.Join(slices.Sorted(maps.Keys(builds)), ", "))
|
||||
ch.consider(ctx, group, c)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// hearFailedBuilds starts awaitRebuilt for the case's verdict when it names failed builds (failedBuilds).
|
||||
func (ch *Checker) hearFailedBuilds(ctx context.Context, group string, cs *Case) {
|
||||
if ch.Rebuilds == nil || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
if v, err := delivery.DecodeVerdict(value); err == nil {
|
||||
if builds := failedBuilds(v, cs); builds != nil {
|
||||
ch.awaitRebuilt(ctx, group, cs.Key, builds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// takeRebuilt says whether every failed build the verdict of vkey names is done again, and forgets it: one retake each.
|
||||
func (ch *Checker) takeRebuilt(vkey string) bool {
|
||||
ch.mu.Lock()
|
||||
defer ch.mu.Unlock()
|
||||
was := ch.rebuilt[vkey]
|
||||
delete(ch.rebuilt, vkey)
|
||||
return was
|
||||
}
|
||||
|
||||
@@ -3,9 +3,12 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -30,6 +33,7 @@ func (f *fakeCompositions) Compose(ctx context.Context, c *Case) (delivery.Compo
|
||||
// 1297/AC3 (its unit half): a check whose tests pass takes the assigner's composition of it, by the check's own key,
|
||||
// into its verdict: a pass says it passed with the mesh-sdk and machines it composed, and is no longer "not run".
|
||||
func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
||||
t.Run("a change to a build module or base names the modules built on it", aChangeToABuildModuleOrBaseNamesTheModulesBuiltOnIt)
|
||||
b := newBench(t)
|
||||
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
|
||||
b.ex.Compositions = f
|
||||
@@ -99,3 +103,165 @@ func TestACompositionNotHadIsNotRun(t *testing.T) {
|
||||
type fixedTrunks struct{ heads map[string]string }
|
||||
|
||||
func (f fixedTrunks) Heads() map[string]string { return f.heads }
|
||||
|
||||
// fakeEnds answers the builder's builds of a candidate as fakeBuilds holds them, recording the builds waited on; the
|
||||
// composition must not have been asked yet when it is (ADR 0603 §10: the check composes with the builds).
|
||||
type fakeEnds struct {
|
||||
builds fakeBuilds
|
||||
asked map[string]string
|
||||
composed *fakeCompositions
|
||||
early bool
|
||||
}
|
||||
|
||||
func (f *fakeEnds) Ended(_ context.Context, builds map[string]string) (map[string]delivery.BuilderBuild, error) {
|
||||
f.asked = builds
|
||||
f.early = f.composed != nil && len(f.composed.asked) > 0
|
||||
out := map[string]delivery.BuilderBuild{}
|
||||
for module, fp := range builds {
|
||||
if b, ok := f.builds[module+" "+fp]; ok && (b.State == delivery.BuildDone || b.State == delivery.BuildFailed) {
|
||||
out[module] = b
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (f *fakeEnds) Done(context.Context, map[string]string) bool { return false }
|
||||
|
||||
// fixedServing are the verbs the running builds serve.
|
||||
type fixedServing []string
|
||||
|
||||
func (f fixedServing) Served(context.Context) (func(string) bool, error) {
|
||||
return func(v string) bool { return slices.Contains(f, v) }, nil
|
||||
}
|
||||
|
||||
// knownRuns are the modules the trunk holds a record of (the indexer's modules): any commit.
|
||||
type knownRuns []string
|
||||
|
||||
func (k knownRuns) CommitOf(module string) string {
|
||||
if slices.Contains(k, module) {
|
||||
return trunkA
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ADR 0603 §10: before it composes, the check waits on the builder's builds of the candidate's changed modules and composes
|
||||
// with them; a failed build fails the check naming it, and one not ended within the bound is named. It does not wait on a
|
||||
// module built by a build module the change alters, nor on one standing on a base the change alters, nor on one the trunk
|
||||
// holds no record of, and names each in the composition's warnings (and so the check's line); a change that alters none of
|
||||
// these says nothing.
|
||||
func aChangeToABuildModuleOrBaseNamesTheModulesBuiltOnIt(t *testing.T) {
|
||||
dig := func(c string) string { return strings.Repeat(c, 64) }
|
||||
fp := "src2:" + dig("f")
|
||||
goBuild := &catalogue.Build{Artifacts: []catalogue.Artifact{{Name: "bin", Kind: catalogue.ArtifactBundle, Language: "go"}}}
|
||||
tsBuild := &catalogue.Build{Artifacts: []catalogue.Artifact{{Name: "app", Kind: catalogue.ArtifactBundle, Language: "typescript"}}}
|
||||
done := delivery.BuilderBuild{State: delivery.BuildDone}
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
manifest map[string]catalogue.Manifest
|
||||
builds map[string]delivery.BuilderBuild // by module, at fp; nil leaves the check without the builds' ends
|
||||
runs knownRuns // nil: the trunk holds every module
|
||||
want []string
|
||||
waited string // the modules waited on, sorted
|
||||
failed string // the failure's test, when the check fails on a build
|
||||
tests bool // the tests fail: the check fails before it would wait
|
||||
served []string // what the running builds serve (ADR 0614 §7); nil reads nothing of what runs
|
||||
}{
|
||||
{name: "the Go build module invokes a verb no running build serves",
|
||||
manifest: map[string]catalogue.Manifest{"mesh-build-go": {Module: "mesh-build-go", Build: goBuild,
|
||||
Invokes: []string{"seat:package-registry.publish", "seat:artifact-store.put"}}, "web": {Module: "web", Build: goBuild}},
|
||||
builds: map[string]delivery.BuilderBuild{"mesh-build-go": done}, served: []string{"seat:artifact-store.put"},
|
||||
want: []string{"the changed build module mesh-build-go invokes seat:package-registry.publish, which no running build serves, " +
|
||||
"so the check cannot run it: web, which it builds, are composed from their manifests (ADR 0614 §7)"}, waited: "mesh-build-go"},
|
||||
{name: "the Go build module changes",
|
||||
manifest: map[string]catalogue.Manifest{"mesh-build-go": {Module: "mesh-build-go", Build: goBuild}, "web": {Module: "web", Build: goBuild},
|
||||
"mail": {Module: "mail", Build: goBuild}, "docs": {Module: "docs"}},
|
||||
builds: map[string]delivery.BuilderBuild{"mesh-build-go": done, "docs": done},
|
||||
want: []string{"the build module mesh-build-go: mail, web, which it builds"}, waited: "docs, mesh-build-go"},
|
||||
{name: "the Node build module changes",
|
||||
manifest: map[string]catalogue.Manifest{"mesh-build-node": {Module: "mesh-build-node", Build: goBuild}, "panel": {Module: "panel", Build: tsBuild},
|
||||
"web": {Module: "web", Build: goBuild}},
|
||||
builds: map[string]delivery.BuilderBuild{"mesh-build-node": done, "web": done},
|
||||
want: []string{"the build module mesh-build-node: panel, which it builds"}, waited: "mesh-build-node, web"},
|
||||
{name: "a base changes",
|
||||
manifest: map[string]catalogue.Manifest{"image": {Module: "image"},
|
||||
"web": {Module: "web", Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: "image", Artifact: "img"}}}}},
|
||||
builds: map[string]delivery.BuilderBuild{"image": done},
|
||||
want: []string{"alters image, a base of web", "waits on the build of image alone"}, waited: "image"},
|
||||
{name: "neither changes", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}},
|
||||
builds: map[string]delivery.BuilderBuild{"web": done}, waited: "web"},
|
||||
{name: "a build fails", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}, "mail": {Module: "mail", Build: goBuild}},
|
||||
builds: map[string]delivery.BuilderBuild{"web": {State: delivery.BuildFailed, Error: "go build: undefined: x"}, "mail": done},
|
||||
waited: "mail, web", failed: "build: web"},
|
||||
{name: "the tests fail", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}},
|
||||
builds: map[string]delivery.BuilderBuild{"web": {State: delivery.BuildBuilding}}, tests: true,
|
||||
want: []string{"composes at once, without waiting on the builds of web"}},
|
||||
{name: "a build does not end within the bound", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}},
|
||||
builds: map[string]delivery.BuilderBuild{"web": {State: delivery.BuildBuilding}},
|
||||
want: []string{"the builds of web did not end within the build-wait-bound"}, waited: "web"},
|
||||
{name: "a module the trunk does not hold", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}, "new": {Module: "new", Build: goBuild}},
|
||||
builds: map[string]delivery.BuilderBuild{"web": done}, runs: knownRuns{"web"},
|
||||
want: []string{"new: the trunk holds no record of it yet"}, waited: "web"},
|
||||
{name: "the builds cannot be heard", manifest: map[string]catalogue.Manifest{"web": {Module: "web", Build: goBuild}},
|
||||
want: []string{"the builds of web cannot be heard"}},
|
||||
} {
|
||||
b := candidateBench(t, true)
|
||||
if row.tests {
|
||||
b.test = func(c stepCall) (any, error) {
|
||||
r := captured[Tested](t, "go-test-failing")
|
||||
r.Level, _ = c.args["level"].(string)
|
||||
return r, nil
|
||||
}
|
||||
}
|
||||
comps := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "x", Machines: []string{"g14"}}}
|
||||
b.ex.Compositions = comps
|
||||
var ends *fakeEnds
|
||||
if row.builds != nil {
|
||||
ends = &fakeEnds{builds: fakeBuilds{}, composed: comps}
|
||||
for module, build := range row.builds {
|
||||
ends.builds[module+" "+fp] = build
|
||||
}
|
||||
b.ex.BuildEnds = ends
|
||||
}
|
||||
if row.runs != nil {
|
||||
b.ex.Runs = row.runs
|
||||
}
|
||||
if row.served != nil {
|
||||
b.ex.Serving = fixedServing(row.served)
|
||||
}
|
||||
read, fps := fixedManifests{}, map[string]delivery.CandidateModule{}
|
||||
for module, m := range row.manifest {
|
||||
d := dig(string(rune('a' + len(read))))
|
||||
read[d], fps[module] = m, delivery.CandidateModule{FP2: fp, Manifest: d}
|
||||
}
|
||||
b.ex.Read = read
|
||||
p := pullOf(catalogueRepository, 9, head1)
|
||||
p.Fps, p.Modules = fps, slices.Sorted(maps.Keys(fps))
|
||||
v := b.judge(t, caseOf(t, p)).Verdict
|
||||
if v.Composition == nil || len(comps.asked) != 1 {
|
||||
t.Fatalf("%s: composed %d times: %+v", row.name, len(comps.asked), v)
|
||||
}
|
||||
if ends != nil {
|
||||
if got := strings.Join(slices.Sorted(maps.Keys(ends.asked)), ", "); got != row.waited || ends.early {
|
||||
t.Errorf("%s: waited on %q (after the composition: %v), want %q before it composes", row.name, got, ends.early, row.waited)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case row.failed != "" && (v.State != delivery.VerdictFail || v.Failed.Test != row.failed || !strings.Contains(v.Failed.Words, "undefined: x") ||
|
||||
!strings.Contains(v.Failed.Words, "the check asks itself again when the build is done")):
|
||||
t.Errorf("%s: the verdict is %s failing %+v, want it failed naming %s, the builder's words and how it is asked again", row.name, v.State, v.Failed, row.failed)
|
||||
case row.tests && v.State != delivery.VerdictFail:
|
||||
t.Errorf("%s: %+v, want the tests' failure", row.name, v)
|
||||
case row.failed == "" && !row.tests && v.State != delivery.VerdictPass:
|
||||
t.Errorf("%s: %+v", row.name, v)
|
||||
}
|
||||
words := compositionWords(v.Composition, v.Took.Composition)
|
||||
for _, w := range row.want {
|
||||
if !strings.Contains(words, w) {
|
||||
t.Errorf("%s: the line says %q, want it to name %q", row.name, words, w)
|
||||
}
|
||||
}
|
||||
if row.want == nil && v.Composition.Words != "" {
|
||||
t.Errorf("%s: warned %q though the check waited on every build of the change", row.name, v.Composition.Words)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,7 +52,12 @@ type EngineCompositions struct {
|
||||
// Manifests are the indexer's trunk manifest digests by module, and Verdicts its refusals of them: the catalogue the
|
||||
// sandbox composes with, before the case's own manifests are laid over.
|
||||
Manifests TrunkManifests
|
||||
Verdicts func() map[string]delivery.ManifestVerdict
|
||||
// Published names the latest published version of every mesh package (Examiner.Published): the composer is built
|
||||
// against them and the group's heads, as a check is (issue 1921); nil names the group's heads alone.
|
||||
Published interface {
|
||||
LatestPackages() delivery.PackageVersions
|
||||
}
|
||||
Verdicts func() map[string]delivery.ManifestVerdict
|
||||
}
|
||||
|
||||
// composerHead is the node-engine member's head of a case, or empty.
|
||||
@@ -147,6 +152,12 @@ func JudgeEngines(group, heads, composerName string, nodes []string, answers map
|
||||
v.Warnings = append(v.Warnings, node+": "+delivery.ComposeNoAnswer)
|
||||
continue
|
||||
}
|
||||
if a.Verdict == delivery.ComposeCandidateNotJudged {
|
||||
// Composed in the sandbox and not judged (ADR 0593 §9): named as no answer, with the composer's words; decided by the
|
||||
// verdict, never by the words (ADR 0601 §4's kind).
|
||||
v.Warnings = append(v.Warnings, node+": "+delivery.ComposeNoAnswer+" ("+a.Refused+")")
|
||||
continue
|
||||
}
|
||||
v.Machines = append(v.Machines, node)
|
||||
if a.Verdict != delivery.ComposeCandidateRefused {
|
||||
continue
|
||||
@@ -214,15 +225,18 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
job := delivery.ComposeJob{Fetch: map[string]any{"repository": nodeEngineRepo, "commit": head}, Dir: delivery.RootPath,
|
||||
job := delivery.ComposeJob{Fetch: withPackages(map[string]any{"repository": nodeEngineRepo, "commit": head}, packagesFor(e.Published, c, nodeEngineRepo)), Dir: delivery.RootPath,
|
||||
Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
|
||||
verdict, err := e.Sandbox.Compose(ctx, job)
|
||||
if err != nil {
|
||||
return a, fmt.Errorf("the composer at the node-engine's head %s did not answer for %s: %w", head, node, err)
|
||||
}
|
||||
out := delivery.ComposeCandidateAnswer{V: a.V, Node: node, Candidate: a.Candidate, Verdict: delivery.ComposeCandidateComposed}
|
||||
notJudged := ""
|
||||
for _, cv := range verdict.Cases {
|
||||
if m, ok := cv.Machines[node]; ok && !m.Composes {
|
||||
if m, ok := cv.Machines[node]; ok && m.NotJudged != "" {
|
||||
notJudged = m.NotJudged // the composer did not judge the node: neither composed nor refused (ADR 0593 §9)
|
||||
} else if ok && !m.Composes {
|
||||
out.Verdict = delivery.ComposeCandidateRefused
|
||||
out.Refused = strings.Join(append(append([]string{m.Reason}, m.Problems...), m.Refusals...), "; ")
|
||||
}
|
||||
@@ -234,5 +248,8 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s
|
||||
}
|
||||
}
|
||||
sort.Slice(out.Modules, func(i, j int) bool { return out.Modules[i].Module < out.Modules[j].Module })
|
||||
if notJudged != "" && out.Verdict == delivery.ComposeCandidateComposed {
|
||||
out.Verdict, out.Refused = delivery.ComposeCandidateNotJudged, notJudged
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -150,7 +151,7 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
Machines: map[string]composer.Machine{node: {Composes: false, Reason: "the new composer refuses it"}}}}}
|
||||
}}
|
||||
ec := EngineCompositions{Asker: e, Placed: func() ([]string, error) { return []string{"anchor"}, nil }, Bound: time.Second,
|
||||
Sandbox: sb, Manifests: digests{"web": "sha256:" + strings.Repeat("c", 64)}}
|
||||
Sandbox: sb, Manifests: digests{"web": "sha256:" + strings.Repeat("c", 64)}, Published: published{}}
|
||||
v, err := ec.Compose(context.Background(), host)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -165,9 +166,32 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
string(req.Input) != string(withInput.Input) || req.Cases[0].Manifests["web"].Digest == "" {
|
||||
t.Errorf("job = %+v, request %+v", job, req)
|
||||
}
|
||||
// Issue 1921: the composer is built at the group's heads, also a member that publishes no package yet.
|
||||
if pkgs, _ := buildstep.PackagesArg(job.Fetch["packages"]); len(pkgs) != 1 || pkgs[delivery.ForgeModulePrefix+"mesh-delivery"] != head1 {
|
||||
t.Errorf("the composer is fetched with the packages %v, want the group's mesh-delivery member at its head %s alone", pkgs, head1)
|
||||
}
|
||||
if v.State != delivery.CompositionFail || v.SDK != head2 || !strings.Contains(v.Words, "the new composer refuses it") {
|
||||
t.Errorf("verdict = %+v", v)
|
||||
}
|
||||
// ADR 0593 §9 (ADR 0601 §4's kind): a node the composer did not judge is named as no answer with its words, by the
|
||||
// contract's field, and the check passes; words in Reason alone are no such mark.
|
||||
sb.err = nil
|
||||
for _, row := range []struct {
|
||||
m composer.Machine
|
||||
pass bool
|
||||
}{
|
||||
{composer.Machine{NotJudged: "its node-engine sends no running state for web"}, true},
|
||||
{composer.Machine{Reason: "not judged: its node-engine sends no running state for web"}, false},
|
||||
} {
|
||||
sb.v = func(node string) composer.Verdict {
|
||||
return composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: "change", Machines: map[string]composer.Machine{node: row.m}}}}
|
||||
}
|
||||
v, err = ec.Compose(context.Background(), host)
|
||||
named := len(v.Warnings) == 1 && strings.Contains(v.Warnings[0], "anchor: "+delivery.ComposeNoAnswer+" (its node-engine sends no running state for web)")
|
||||
if err != nil || (v.State == delivery.CompositionPass) != row.pass || named != row.pass {
|
||||
t.Errorf("%+v: verdict %+v, %v", row.m, v, err)
|
||||
}
|
||||
}
|
||||
// Review finding 3 (ADR 0593 §9 lets only a node that does not answer pass): a composer at the head that cannot
|
||||
// run fails the composition, naming the head, never passing as a silent node.
|
||||
sb.err = errors.New("the build of mesh-compose failed")
|
||||
|
||||
@@ -20,6 +20,8 @@ import (
|
||||
|
||||
gobuild "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -1000,6 +1002,17 @@ func (k keptList) PredefinedList(context.Context) (json.RawMessage, error) {
|
||||
return json.RawMessage(k), nil
|
||||
}
|
||||
|
||||
// fixedManifests are the indexer's kept manifests by digest as a test gives them.
|
||||
type fixedManifests map[string]catalogue.Manifest
|
||||
|
||||
func (f fixedManifests) Manifest(_ context.Context, digest string) (catalogue.Manifest, error) {
|
||||
m, ok := f[digest]
|
||||
if !ok {
|
||||
return catalogue.Manifest{}, errors.New("no manifest " + digest)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// fixedDigests are the indexer's manifest digests as a test gives them.
|
||||
type fixedDigests map[string]string
|
||||
|
||||
@@ -1017,17 +1030,75 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
|
||||
// untouched is the judge's refusal of mail, which the change leaves as it is, and trunkMail its refusal on the trunk:
|
||||
// a problem the trunk had is a warning, one the change causes fails (ADR 0579 §2)
|
||||
untouched, trunkMail string
|
||||
// cycle reads the manifests, so the change's module graph is worked out (ADR 0603 §2): with "provisions" the change's
|
||||
// web awaits mail's site while mail awaits web's database, and trunkCycle has the trunk's web await it too; with "bases"
|
||||
// each is built on the other; with "verbs" each invokes the other's tool, and with "a verb and a provision" web invokes
|
||||
// mail's tool while mail awaits web's database: both left out, so neither is refused (ADR 0614 §3, §6).
|
||||
cycle string
|
||||
trunkCycle bool
|
||||
// indexer makes web the holder of the seat mesh-catalog, and the trunk's manifest validator answers for it: "held" with no verdict,
|
||||
// "aside" with a field it does not know; ownJudge has the group change the judge, so the manifest is also judged by the
|
||||
// trunk's, in a second ask (ADR 0633 clause 6, 1585/T22, T23)
|
||||
indexer string
|
||||
ownJudge bool
|
||||
// try judges the same change as a whole group try, whose manifests come from the indexer's candidate of the try's
|
||||
// group (issue 1921): the same judge, the same verdicts.
|
||||
try bool
|
||||
}{
|
||||
{name: "accepted"},
|
||||
{name: "accepted in a group try", try: true},
|
||||
{name: "refused in a group try", try: true, refused: "web has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web has no id`},
|
||||
{name: "a cycle the group try makes", try: true, cycle: "provisions", fails: "the change's module graph has the cycle mail -> web -> mail"},
|
||||
{name: "the indexer accepted by the trunk's manifest validator", indexer: "ok"},
|
||||
{name: "the indexer held by the trunk's manifest validator", indexer: "held", fails: "the indexer web, the holder of the seat mesh-catalog, would never run: the trunk's manifest validator at mesh-delivery's trunk " + strings.Repeat("d", 40) + " does not accept its manifest: the manifest validator holds it, with no verdict yet"},
|
||||
{name: "the indexer with a field the trunk's manifest validator does not know", indexer: "aside", fails: "the manifest validator does not know its fields new_field, so it waits"},
|
||||
{name: "the indexer refused by the trunk's manifest validator", indexer: "refused", refused: "web has no id", fails: "does not accept its manifest: web has no id"},
|
||||
{name: "the indexer under a changed judge is judged by the trunk's as well", indexer: "aside", ownJudge: true, fails: "the manifest validator does not know its fields new_field, so it waits"},
|
||||
{name: "a touched manifest the judge holds", indexer: "heldplain", fails: "holds the manifest of web: it gave no verdict"},
|
||||
{name: "an untouched manifest's problem the trunk had", untouched: "mail says no credential form", trunkMail: "mail says no credential form"},
|
||||
{name: "a problem the change causes in an untouched manifest", untouched: "mail reads a state web no longer keeps",
|
||||
fails: "refuses the manifest of mail: mail reads a state web no longer keeps"},
|
||||
{name: "a cycle the change makes", cycle: "provisions", fails: "the change's module graph has the cycle mail -> web -> mail"},
|
||||
{name: "a cycle the trunk has", cycle: "provisions", trunkCycle: true},
|
||||
{name: "a cycle of bases", cycle: "bases", fails: "the change's module graph has the cycle mail -> web -> mail"},
|
||||
{name: "a cycle of verbs", cycle: "verbs"},
|
||||
{name: "a cycle of a verb and a provision", cycle: "a verb and a provision"},
|
||||
{name: "refused", refused: "web requires \"amqp\"\nweb has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web requires "amqp"; web has no id`},
|
||||
} {
|
||||
b := newBench(t)
|
||||
b.ex.Manifests = fixedDigests{"mail": strings.Repeat("a", 64), "web": strings.Repeat("b", 64)}
|
||||
b.ex.Trunks = fixedTrunks{heads: map[string]string{DeliveryRepository: trunk}}
|
||||
b.ex.Predefined = keptList(`{"font-size": {"type": "number", "description": "the size"}}`)
|
||||
if row.cycle != "" {
|
||||
offer := func(name string) []catalogue.Offer {
|
||||
return []catalogue.Offer{{Name: name, Scope: catalogue.ScopeMesh, AnsweredBy: catalogue.AnsweredByHelper}}
|
||||
}
|
||||
on := func(module string) *catalogue.Build {
|
||||
return &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: module, Artifact: "img"}}}
|
||||
}
|
||||
trunkWeb := catalogue.Manifest{Module: "web", Provides: offer("database")}
|
||||
if row.trunkCycle {
|
||||
trunkWeb.Requires = []string{"site"}
|
||||
}
|
||||
mail := catalogue.Manifest{Module: "mail", Provides: offer("site"), Requires: []string{"database"}}
|
||||
changed := catalogue.Manifest{Module: "web", Provides: offer("database"), Requires: []string{"site"}}
|
||||
switch row.cycle {
|
||||
case "bases":
|
||||
mail = catalogue.Manifest{Module: "mail", Build: on("web")}
|
||||
changed = catalogue.Manifest{Module: "web", Build: on("mail")}
|
||||
case "verbs":
|
||||
mail = catalogue.Manifest{Module: "mail", Tools: []string{"mail_send"}, Invokes: []string{"web.web_page"}}
|
||||
changed = catalogue.Manifest{Module: "web", Tools: []string{"web_page"}, Invokes: []string{"mail.mail_send"}}
|
||||
case "a verb and a provision":
|
||||
mail = catalogue.Manifest{Module: "mail", Tools: []string{"mail_send"}, Requires: []string{"database"}}
|
||||
changed = catalogue.Manifest{Module: "web", Provides: offer("database"), Invokes: []string{"mail.mail_send"}}
|
||||
}
|
||||
b.ex.Read = fixedManifests{strings.Repeat("a", 64): mail, strings.Repeat("b", 64): trunkWeb, web: changed}
|
||||
}
|
||||
if row.indexer != "" && row.indexer != "heldplain" {
|
||||
b.ex.Read = fixedManifests{strings.Repeat("a", 64): {Module: "mail"}, strings.Repeat("b", 64): {Module: "web"},
|
||||
web: {Module: "web", Claims: []catalogue.Claim{{Name: CatalogSeat}}}}
|
||||
}
|
||||
var judged []delivery.ComposeJob
|
||||
answer := b.asks.Answer
|
||||
b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||
@@ -1036,6 +1107,12 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
|
||||
}
|
||||
judged = append(judged, *a.Compose)
|
||||
read := composer.ManifestRead{Accepted: row.refused == "", Refused: row.refused}
|
||||
switch row.indexer {
|
||||
case "held", "heldplain":
|
||||
read = composer.ManifestRead{}
|
||||
case "aside":
|
||||
read.SetAside = "new_field"
|
||||
}
|
||||
verdict, _ := json.Marshal(composer.Verdict{V: composer.Version, SDK: "exe:1", Cases: []composer.CaseVerdict{{Name: JudgeCase,
|
||||
Manifests: map[string]composer.ManifestRead{"web": read, "mail": {Accepted: row.untouched == "", Refused: row.untouched}}},
|
||||
{Name: JudgeTrunk, Manifests: map[string]composer.ManifestRead{"web": {Accepted: true}, "mail": {Accepted: row.trunkMail == "", Refused: row.trunkMail}}}}})
|
||||
@@ -1043,9 +1120,53 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
|
||||
return delivery.Step{ID: a.ID, State: delivery.StepPassed, Result: result}
|
||||
}
|
||||
p := pullOf(repoA, 7, head1)
|
||||
if row.ownJudge {
|
||||
p = pullOf(DeliveryRepository, 7, head1)
|
||||
b.fetch.Changed = []string{JudgeDir + "/judge/judge.go"}
|
||||
}
|
||||
p.Modules = []string{"web"}
|
||||
p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}}
|
||||
v := b.judge(t, caseOf(t, p)).Verdict
|
||||
// Issue 1921: the judge is built at the group's heads too, also a member that publishes no package yet.
|
||||
cs := caseOf(t, p, pullOf("novox/mesh-networking", 9, head2))
|
||||
if row.try {
|
||||
cs = TryCase("try-1", Try{Members: []TryMember{{Repository: repoA, Head: head1, Trunk: trunkA}}}, nil)
|
||||
b.ex.Candidates = &fakeTryCandidates{cand: delivery.Candidate{FPSet: "fps", Pulls: []delivery.CandidatePull{p}}}
|
||||
}
|
||||
v := b.judge(t, cs).Verdict
|
||||
if row.indexer == "ok" && !slices.ContainsFunc(cs.notes, func(n string) bool {
|
||||
return strings.Contains(n, "the new indexer's manifest of web is accepted by the trunk's manifest validator")
|
||||
}) {
|
||||
t.Errorf("%s: the case says %v, want the new indexer's manifest said accepted by the trunk's manifest validator", row.name, cs.notes)
|
||||
}
|
||||
// Issue 1921: the judge of the change is built at the group's heads (also a member that publishes no package yet), but
|
||||
// the trunk's manifest validator that rules on a new indexer is built at the latest published versions alone, never
|
||||
// at a group's head (ADR 0633 clause 6): a group with a member of its own then judges the indexer a second time.
|
||||
trunkRuns := 0
|
||||
if row.indexer != "" && row.indexer != "heldplain" && (row.ownJudge || !row.try) {
|
||||
trunkRuns = 1
|
||||
}
|
||||
if want := 1 + trunkRuns; len(judged) != want {
|
||||
t.Fatalf("%s: %d judge asks %+v, want %d", row.name, len(judged), judged, want)
|
||||
}
|
||||
for i, j := range judged {
|
||||
pkgs, _ := buildstep.PackagesArg(j.Fetch["packages"])
|
||||
atGroup := pkgs[delivery.ForgeModulePrefix+"mesh-networking"] == head2
|
||||
switch trunkRun := trunkRuns == 1 && i == len(judged)-1; {
|
||||
case row.try:
|
||||
case trunkRun && atGroup:
|
||||
t.Errorf("%s: the trunk's manifest validator is built with the group's head: %v", row.name, pkgs)
|
||||
case !trunkRun && !atGroup:
|
||||
t.Errorf("%s: the judge at %v is fetched with the packages %v, want the group's mesh-networking member at its head %s", row.name, j.Fetch["commit"], pkgs, head2)
|
||||
}
|
||||
}
|
||||
if row.ownJudge && (len(judged) != 2 || judged[0].Fetch["commit"] != head1 || judged[1].Fetch["commit"] != trunk) {
|
||||
t.Fatalf("%s: judged %+v, want the head's judge and then the trunk's", row.name, judged)
|
||||
}
|
||||
if row.ownJudge {
|
||||
judged = judged[1:]
|
||||
} else if trunkRuns == 1 {
|
||||
judged = judged[1:] // the group's run; the trunk's is the one read below
|
||||
}
|
||||
if len(judged) != 1 || judged[0].Fetch["repository"] != DeliveryRepository || judged[0].Fetch["commit"] != trunk ||
|
||||
judged[0].Package != JudgePackage || judged[0].Dir != JudgeDir {
|
||||
t.Fatalf("%s: judged %+v (verdict %+v)", row.name, judged, v)
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
)
|
||||
|
||||
// The check refuses a change whose module graph has a cycle and names it (novox/hq ADR 0603 §2): modules that would each
|
||||
// wait for the other for good, through their bases, the provisions and bus state they await, or the verb edges the graph
|
||||
// keeps (ADR 0614 §6). A verb edge on a cycle is left out of the graph, so it is never refused (§3), and a package or a
|
||||
// build module's edge is a trigger, never a wait, so a cycle through one is not refused either (§6). The graph is worked out from the
|
||||
// catalogue as the change leaves it (every trunk manifest, each one a pull request touches replaced by its candidate's),
|
||||
// with the contract's ManifestGraph, the way the indexer works out its graph's edges. A cycle the trunk already has among
|
||||
// the same modules is the trunk's: said in the verdict's notes, not the change's failure.
|
||||
|
||||
// ManifestReader reads a kept manifest by its digest (the indexer's manifests state).
|
||||
type ManifestReader interface {
|
||||
Manifest(ctx context.Context, digest string) (catalogue.Manifest, error)
|
||||
}
|
||||
|
||||
// refuseCycle fails the check for a cycle of the change's module graph, naming it; nil when the case touches no manifest,
|
||||
// when nothing reads manifests, or when there is none. A manifest that cannot be read fails the check, said, never passes it.
|
||||
func (st *state) refuseCycle(ctx context.Context) *delivery.Verdict {
|
||||
if st.j.Manifests == nil || st.j.Read == nil || st.narrowTry() {
|
||||
return nil
|
||||
}
|
||||
trunk := st.j.Manifests.ManifestDigests()
|
||||
req, touched := judgeRequest(trunk, st.c.Pulls)
|
||||
if len(touched) == 0 {
|
||||
return nil
|
||||
}
|
||||
read := func(digests map[string]composer.Manifest) (map[string]catalogue.Manifest, error) {
|
||||
out := map[string]catalogue.Manifest{}
|
||||
for _, module := range slices.Sorted(maps.Keys(digests)) {
|
||||
m, err := st.j.Read.Manifest(ctx, digests[module].Digest)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the manifest %s of %s: %w", digests[module].Digest, module, err)
|
||||
}
|
||||
out[module] = m
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
change, err := read(req.Cases[0].Manifests)
|
||||
if err != nil {
|
||||
return fail(goModule, "the change's module graph cannot be worked out, so no cycle can be ruled out: "+err.Error())
|
||||
}
|
||||
cycle := graph.OfManifests(change).Cycle()
|
||||
if cycle == nil {
|
||||
st.c.note("the change's module graph has no cycle")
|
||||
return nil
|
||||
}
|
||||
words := strings.Join(cycle, " -> ")
|
||||
if before, err := read(req.Cases[1].Manifests); err == nil {
|
||||
among := graph.Graph{}
|
||||
for name, n := range graph.OfManifests(before) {
|
||||
if slices.Contains(cycle, name) {
|
||||
among[name] = n
|
||||
}
|
||||
}
|
||||
if among.Cycle() != nil {
|
||||
st.c.note("the module graph has the cycle " + words + ", which the trunk already has among the same modules (ADR 0603 §2)")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fail(goModule, fmt.Sprintf("the change's module graph has the cycle %s: these modules would each wait for the other for "+
|
||||
"good, through their bases, the provisions and bus state they await, or the verbs they invoke (ADR 0603 §2, ADR 0614 §6)", words))
|
||||
}
|
||||
@@ -102,6 +102,11 @@ type Examiner struct {
|
||||
// Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing
|
||||
// of the builds.
|
||||
Builds BuildStates
|
||||
// BuildEnds waits on the builder's builds of the candidate before the check composes (ADR 0603 §10); nil composes
|
||||
// without waiting, said in the composition's warnings.
|
||||
BuildEnds BuildEnds
|
||||
// BuildWait is how long the check waits for those builds (the setting build-wait-bound); DefaultBuildWait when 0.
|
||||
BuildWait time.Duration
|
||||
// Announce says a preview was written under the key (EventPreviewWritten); nil says nothing.
|
||||
Announce Announcer
|
||||
// Packages names, of a repository, the paths (repository-relative) its TypeScript package artifacts are built from, from
|
||||
@@ -114,6 +119,15 @@ type Examiner struct {
|
||||
// Manifests are the indexer's manifest digests by module: the catalogue a change's manifests are judged in by the
|
||||
// indexer's judge (manifests_judge.go, novox/hq ADR 0578 §2); nil judges no manifest.
|
||||
Manifests TrunkManifests
|
||||
// Read reads a kept manifest by its digest: the change's module graph is worked out from them (graph_cycle.go, novox/hq
|
||||
// ADR 0603 §2); nil works out no graph.
|
||||
Read ManifestReader
|
||||
// Serving says, from what runs, whether a running healthy build serves a seat verb or module tool (novox/hq ADR 0614
|
||||
// §7): the node-engines' haves, the builder's build of each, its manifest (graph.Served). Nil reads nothing, and no
|
||||
// changed build module is named for a verb it invokes.
|
||||
Serving interface {
|
||||
Served(ctx context.Context) (func(invoke string) bool, error)
|
||||
}
|
||||
// Predefined reads the catalogue's predefined keys the indexer keeps (manifests_judge.go); nil reads none.
|
||||
Predefined PredefinedList
|
||||
// Compositions makes a check's composition (ADR 0593 §7): the engines' answers (EngineCompositions); nil leaves the
|
||||
@@ -125,8 +139,18 @@ type Examiner struct {
|
||||
Nodes interface {
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
}
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
// Candidates answers the indexer's candidate of a try's group once it wrote it from the try's previews, the manifests
|
||||
// and fingerprints a group try's manifest validator and composition judge (issue 1921); nil leaves a whole try with no
|
||||
// manifest or composition judged, said.
|
||||
Candidates TryCandidates
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// TryCandidates answers the candidate the indexer holds for a try's group at its heads hash, waiting for it within the
|
||||
// bound; an error says it did not come (and why).
|
||||
type TryCandidates interface {
|
||||
TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error)
|
||||
}
|
||||
|
||||
// CompositionReader makes a check's composition verdict; an error is only a case that cannot be composed at all.
|
||||
@@ -232,6 +256,50 @@ type state struct {
|
||||
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
|
||||
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
|
||||
unpreviewed []string
|
||||
// noCandidate says why a whole try has no candidate of its group from the indexer, so its manifests were not judged and
|
||||
// it was not composed; empty when it has one, and for a pull request's check.
|
||||
noCandidate string
|
||||
}
|
||||
|
||||
// narrowTry says the case is a try narrowed to a level or to some tests: it judges no manifest, graph or composition.
|
||||
func (st *state) narrowTry() bool { return st.c.Try != nil && !st.c.Try.Whole() }
|
||||
|
||||
// takeTryCandidate gives a whole try's pulls the fingerprints and manifests the indexer's candidate of its group holds
|
||||
// (the group's pull-request check has them from its candidate, CaseOf), once the indexer wrote it from the try's previews.
|
||||
// A try with no such candidate is said so, in a note and in its composition, and is no pass (Tryer.run).
|
||||
func (st *state) takeTryCandidate(ctx context.Context) {
|
||||
if st.c.Try == nil || !st.c.Try.Whole() {
|
||||
return
|
||||
}
|
||||
no := func(why string) {
|
||||
st.noCandidate = "the group's manifests and composition were not judged: " + why
|
||||
st.c.note(st.noCandidate)
|
||||
}
|
||||
if st.j.Candidates == nil {
|
||||
no("the checker reads no candidate of the indexer's")
|
||||
return
|
||||
}
|
||||
if len(st.unpreviewed) > 0 {
|
||||
no("no preview of a merge of " + strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer)")
|
||||
return
|
||||
}
|
||||
if st.c.Wait != nil {
|
||||
st.c.Wait("the indexer's candidate of the group at its heads (up to " + st.j.buildWait().String() + ")")
|
||||
defer st.c.Wait("")
|
||||
}
|
||||
cand, err := st.j.Candidates.TryCandidate(ctx, st.c.Group, st.c.HeadsHash, st.j.buildWait())
|
||||
if err != nil {
|
||||
no(err.Error())
|
||||
return
|
||||
}
|
||||
for i, p := range st.c.Pulls {
|
||||
for _, cp := range cand.Pulls {
|
||||
if cp.Repository == p.Repository && cp.Head.Head == p.Head {
|
||||
st.c.Pulls[i].Modules, st.c.Pulls[i].Fps = cp.Modules, cp.Fps
|
||||
}
|
||||
}
|
||||
}
|
||||
st.c.FPSet = cand.FPSet
|
||||
}
|
||||
|
||||
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
||||
@@ -242,7 +310,8 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
||||
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
|
||||
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
|
||||
tests := st.judgeAll(ctx)
|
||||
if v := settled(joinFailures(tests, st.judgeManifests(ctx))); v != nil {
|
||||
st.takeTryCandidate(ctx)
|
||||
if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil {
|
||||
return st.finish(ctx, *v), nil
|
||||
}
|
||||
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
|
||||
@@ -268,7 +337,14 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
|
||||
return
|
||||
}
|
||||
if st.j.Compositions == nil || st.c.Try != nil {
|
||||
if st.narrowTry() {
|
||||
return
|
||||
}
|
||||
if st.noCandidate != "" {
|
||||
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: st.noCandidate}
|
||||
return
|
||||
}
|
||||
if st.j.Compositions == nil {
|
||||
return
|
||||
}
|
||||
if len(st.unpreviewed) > 0 {
|
||||
@@ -277,6 +353,7 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer), so the change cannot be composed"}
|
||||
return
|
||||
}
|
||||
words := st.awaitBuilds(ctx, v) // ADR 0603 §10: the check composes with the candidate's builds
|
||||
comp, err := st.j.Compositions.Compose(ctx, st.c)
|
||||
if err != nil {
|
||||
// Said in the verdict, never left as a composition not yet run (review finding 4).
|
||||
@@ -284,6 +361,7 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: "not made: " + err.Error()}
|
||||
return
|
||||
}
|
||||
comp.Warnings = append(comp.Warnings, words...) // ADR 0603 §10
|
||||
withComposition(v, comp)
|
||||
if v.Composition != nil && v.Composition.State != delivery.CompositionNotRun {
|
||||
v.Composition.FPSet = st.c.FPSet // a carry keeps it while the fingerprints stay the same (ADR 0548 §5)
|
||||
@@ -417,7 +495,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
// Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests
|
||||
// failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0).
|
||||
switch {
|
||||
case st.c.Try != nil: // a try owes the indexer no preview: no pull request holds its head
|
||||
case st.narrowTry(): // a try of a level or some tests owes the indexer no preview: it judges no manifest
|
||||
case len(merged) == len(modules):
|
||||
st.j.writePreview(ctx, p, merged)
|
||||
default:
|
||||
@@ -546,7 +624,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch))
|
||||
}
|
||||
if len(t.fetched.Conflicts) > 0 {
|
||||
if st.c.Try == nil {
|
||||
if !st.narrowTry() {
|
||||
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
||||
}
|
||||
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
|
||||
|
||||
@@ -97,7 +97,7 @@ func judgeRequest(trunk map[string]string, pulls []Pull) (req composer.Request,
|
||||
// refuses. A case that touches no manifest, or an examiner that reads no trunk manifests, judges nothing; a judge that
|
||||
// cannot be asked or does not answer fails the check, said, never passes it.
|
||||
func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
|
||||
if st.j.Manifests == nil || st.c.Try != nil {
|
||||
if st.j.Manifests == nil || st.narrowTry() {
|
||||
return nil
|
||||
}
|
||||
req, touched := judgeRequest(st.j.Manifests.ManifestDigests(), st.c.Pulls)
|
||||
@@ -120,26 +120,12 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
|
||||
req.Predefined = list
|
||||
st.c.note("the manifests are judged against the predefined keys of " + from)
|
||||
}
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return fail(goModule, err.Error())
|
||||
cv, trunkCase, v := st.runJudge(ctx, req, touched, commit, at, st.packagesOf(DeliveryRepository))
|
||||
if v != nil {
|
||||
return v
|
||||
}
|
||||
job := delivery.ComposeJob{Fetch: map[string]any{"repository": DeliveryRepository, "commit": commit}, Dir: JudgeDir,
|
||||
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
|
||||
At: st.j.now().UTC(), Compose: &job}
|
||||
st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "")
|
||||
st.c.Asked(a.ID)
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error())
|
||||
}
|
||||
step, err := st.j.Asks.Wait(ctx, a)
|
||||
if err != nil {
|
||||
return fail(goModule, "the judge of the change's manifests: "+err.Error())
|
||||
}
|
||||
cv, trunkCase, why := judgeAnswer(step)
|
||||
if why != "" {
|
||||
return fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why))
|
||||
if v := st.refuseUnmovableIndexer(ctx, req, touched, cv, at); v != nil {
|
||||
return v
|
||||
}
|
||||
// A new or changed manifest is judged in full; of one the change leaves as it is, a problem the trunk already had is a
|
||||
// warning (it is not refused for a rule it already broke), and a problem the change causes in it, which the trunk case
|
||||
@@ -147,7 +133,13 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
|
||||
var refused, warned []string
|
||||
for _, module := range slices.Sorted(maps.Keys(cv.Manifests)) {
|
||||
r := cv.Manifests[module]
|
||||
if r.Accepted || r.Refused == "" {
|
||||
if r.Accepted {
|
||||
continue
|
||||
}
|
||||
if r.Refused == "" { // held: a verdict not given is not one given (ADR 0633 clause 6)
|
||||
if slices.Contains(touched, module) {
|
||||
refused = append(refused, fmt.Sprintf("the indexer's judge at %s holds the manifest of %s: it gave no verdict", at, module))
|
||||
}
|
||||
continue
|
||||
}
|
||||
had := map[string]bool{}
|
||||
@@ -183,6 +175,100 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
|
||||
return nil
|
||||
}
|
||||
|
||||
// runJudge asks the Go build seat to build the judge at the commit and run it over the request, and reads its change and
|
||||
// trunk cases; a failed verdict when it cannot be asked or does not answer.
|
||||
func (st *state) runJudge(ctx context.Context, req composer.Request, touched []string, commit, at string, versions delivery.PackageVersions) (change, trunk composer.CaseVerdict, v *delivery.Verdict) {
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return change, trunk, fail(goModule, err.Error())
|
||||
}
|
||||
job := delivery.ComposeJob{Fetch: withPackages(map[string]any{"repository": DeliveryRepository, "commit": commit}, versions), Dir: JudgeDir,
|
||||
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
|
||||
At: st.j.now().UTC(), Compose: &job}
|
||||
st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "")
|
||||
st.c.Asked(a.ID)
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return change, trunk, fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error())
|
||||
}
|
||||
step, err := st.j.Asks.Wait(ctx, a)
|
||||
if err != nil {
|
||||
return change, trunk, fail(goModule, "the judge of the change's manifests: "+err.Error())
|
||||
}
|
||||
change, trunk, why := judgeAnswer(step)
|
||||
if why != "" {
|
||||
return change, trunk, fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why))
|
||||
}
|
||||
return change, trunk, nil
|
||||
}
|
||||
|
||||
// CatalogSeat is the seat the indexer holds: the module whose manifest claims it is the indexer, found by the claim and
|
||||
// never by its name (novox/hq ADR 0633 clause 6, ADR 0601 §4).
|
||||
const CatalogSeat = "mesh-catalog"
|
||||
|
||||
// refuseUnmovableIndexer fails the check when the trunk's manifest validator, the one the running parts hold, would
|
||||
// refuse or hold the change's new indexer manifest: such an indexer would never run, and every revision behind it would
|
||||
// wait for ever (ADR 0633 clause 6). The indexer is the touched module whose manifest claims the seat mesh-catalog.
|
||||
// When the change judges with its own head's manifest validator (it changes it), the manifest is judged by the trunk's
|
||||
// as well. The verdicts read are the trunk's manifest validator's: its change case, where refused, held (not accepted
|
||||
// with no reason) or with a field set aside (a field the running manifest validator does not know waits, ADR 0601 §3)
|
||||
// each fail it, naming the module and the problems.
|
||||
func (st *state) refuseUnmovableIndexer(ctx context.Context, req composer.Request, touched []string, cv composer.CaseVerdict, judgedAt string) *delivery.Verdict {
|
||||
if st.j.Read == nil {
|
||||
return nil
|
||||
}
|
||||
var indexer string
|
||||
for _, module := range touched {
|
||||
m, err := st.j.Read.Manifest(ctx, req.Cases[0].Manifests[module].Digest)
|
||||
if err != nil {
|
||||
return fail(goModule, fmt.Sprintf("the manifest of %s cannot be read, so the indexer cannot be ruled out: %v", module, err))
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == CatalogSeat {
|
||||
indexer = module
|
||||
}
|
||||
}
|
||||
}
|
||||
if indexer == "" {
|
||||
return nil
|
||||
}
|
||||
// The validator the running parts hold is the trunk's, built at the latest published versions alone, never at the group's
|
||||
// heads (ADR 0633 clause 6): a group's mesh-sdk or mesh-contracts head must not change the validator that judges the new
|
||||
// indexer. The head's run is reused when it was that validator already.
|
||||
if trunkVersions := st.trunkJudgeVersions(); st.judgeHead != "" || !maps.Equal(st.packagesOf(DeliveryRepository), trunkVersions) {
|
||||
trunk := ""
|
||||
if st.j.Trunks != nil {
|
||||
trunk = st.j.Trunks.Heads()[DeliveryRepository]
|
||||
}
|
||||
if trunk == "" {
|
||||
return fail(goModule, "the new indexer cannot be judged by the trunk's manifest validator: the trunk head of "+DeliveryRepository+" is not known")
|
||||
}
|
||||
var v *delivery.Verdict
|
||||
judgedAt = "mesh-delivery's trunk " + trunk
|
||||
if cv, _, v = st.runJudge(ctx, req, touched, trunk, judgedAt, trunkVersions); v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
r, ok := cv.Manifests[indexer]
|
||||
var problems []string
|
||||
switch {
|
||||
case !ok:
|
||||
problems = append(problems, "the manifest validator gave no verdict")
|
||||
case r.Refused != "":
|
||||
problems = append(problems, strings.Split(r.Refused, "\n")...)
|
||||
case !r.Accepted:
|
||||
problems = append(problems, "the manifest validator holds it, with no verdict yet")
|
||||
case r.SetAside != "":
|
||||
problems = append(problems, "the manifest validator does not know its fields "+r.SetAside+", so it waits")
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fail(goModule, fmt.Sprintf("the indexer %s, the holder of the seat %s, would never run: the trunk's manifest validator at %s does not accept its manifest: %s",
|
||||
indexer, CatalogSeat, judgedAt, strings.Join(problems, "; ")))
|
||||
}
|
||||
st.c.note(fmt.Sprintf("the new indexer's manifest of %s is accepted by the trunk's manifest validator at %s", indexer, judgedAt))
|
||||
return nil
|
||||
}
|
||||
|
||||
// judgeAnswer reads the judge's change and trunk cases from its step, or why it cannot.
|
||||
func judgeAnswer(step delivery.Step) (change, trunk composer.CaseVerdict, why string) {
|
||||
if step.State != delivery.StepPassed {
|
||||
@@ -257,3 +343,9 @@ func (st *state) judgedByMember(p Pull, merged []Fetched) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// trunkJudgeVersions are the versions the trunk's manifest validator is built at: the latest published version of each
|
||||
// package, with no group member's head (ADR 0633 clause 6).
|
||||
func (st *state) trunkJudgeVersions() delivery.PackageVersions {
|
||||
return packagesFor(st.j.Published, &Case{}, DeliveryRepository)
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/grouppackages"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -31,37 +31,28 @@ type Forge interface {
|
||||
File(ctx context.Context, repository, path, ref string) (string, error)
|
||||
}
|
||||
|
||||
// packagesOf is the version of each mesh package a Go check of repository builds against (ADR 0573 §2, §4): the latest
|
||||
// published version of each package the mesh's modules use (Published), and for a package one of the case's pull
|
||||
// requests publishes, that pull request's head (mesh-sdk's Go module whenever mesh-sdk is a member). A package of
|
||||
// repository itself comes with the tree the check fetched, and is left out. Nil for none.
|
||||
// packagesOf is the version of each mesh package a Go build of repository (a check ask, the judge, the composer) builds
|
||||
// against (ADR 0573 §2, §4): grouppackages.Heads over the latest published version of each package the mesh's modules use
|
||||
// (Published) and the heads of the case's pull requests, so a member that publishes no package yet is named too (issue
|
||||
// 1921). Nil for none.
|
||||
func (st *state) packagesOf(repository string) delivery.PackageVersions {
|
||||
versions := delivery.PackageVersions{}
|
||||
if st.j.Published != nil {
|
||||
maps.Copy(versions, st.j.Published.LatestPackages())
|
||||
return packagesFor(st.j.Published, st.c, repository)
|
||||
}
|
||||
|
||||
// packagesFor is packagesOf for a case and the published versions: every producer of a build at a group's heads takes its
|
||||
// versions from here, so none leaves a member out.
|
||||
func packagesFor(published interface {
|
||||
LatestPackages() delivery.PackageVersions
|
||||
}, c *Case, repository string) delivery.PackageVersions {
|
||||
var latest delivery.PackageVersions
|
||||
if published != nil {
|
||||
latest = published.LatestPackages()
|
||||
}
|
||||
if st.c.member(SDKRepository) != nil {
|
||||
if _, named := versions[SDKModulePath]; !named {
|
||||
versions[SDKModulePath] = ""
|
||||
}
|
||||
heads := map[string]string{}
|
||||
for _, m := range c.Pulls {
|
||||
heads[m.Repository] = m.Head
|
||||
}
|
||||
for pkg := range versions {
|
||||
repo, _, ok := delivery.PackageRepository(pkg)
|
||||
if !ok || delivery.SameRepository(repo, repository) {
|
||||
delete(versions, pkg)
|
||||
continue
|
||||
}
|
||||
if m := st.c.member(repo); m != nil && len(m.Head) == 40 {
|
||||
versions[pkg] = m.Head
|
||||
}
|
||||
if versions[pkg] == "" {
|
||||
delete(versions, pkg)
|
||||
}
|
||||
}
|
||||
if len(versions) == 0 {
|
||||
return nil
|
||||
}
|
||||
return versions
|
||||
return grouppackages.Heads(latest, heads, repository)
|
||||
}
|
||||
|
||||
// unheardWords ends the words of a failed read of the forge that reached nobody: the forge was not heard (UnheardWords).
|
||||
@@ -80,3 +71,17 @@ func forgeMachine(err error, node string) string {
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// withPackages puts the versions into a fetch's arguments (the fetch tool's packages argument, which a compose ask has no
|
||||
// field for: a compose ask names none, StepAsk.Packages), so the composer or the judge is built at the group's heads.
|
||||
func withPackages(fetch map[string]any, versions delivery.PackageVersions) map[string]any {
|
||||
if len(versions) == 0 {
|
||||
return fetch
|
||||
}
|
||||
packages := make(map[string]any, len(versions))
|
||||
for pkg, version := range versions {
|
||||
packages[pkg] = version
|
||||
}
|
||||
fetch["packages"] = packages
|
||||
return fetch
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/grouppackages"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -60,6 +61,48 @@ func TestAGroupMemberIsBuiltAgainstTheSDKMembersHeadWhateverItsGoModNames(t *tes
|
||||
if got := packagesOfFetch(t, b, "novox/mesh-catalog"); len(got) != 2 || got[SDKModulePath] != trunkS || got[contract] != trunkA {
|
||||
t.Errorf("a pull request alone is built against %v, want every package at its latest published version", got)
|
||||
}
|
||||
// A member that publishes no package yet is still named, by its root module path, at its head (issue 1921).
|
||||
b = newBench(t)
|
||||
b.ex.Published = published{}
|
||||
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf("novox/mesh-host", 8, head2))).Verdict; v.State != delivery.VerdictPass {
|
||||
t.Fatalf("an unpublished member: verdict = %+v", v)
|
||||
}
|
||||
if got := packagesOfFetch(t, b, repoA); len(got) != 1 || got["git.novox.be/novox/mesh-host"] != head2 {
|
||||
t.Errorf("the member's packages = %v, want the unpublished member's root at its head %s alone", got, head2)
|
||||
}
|
||||
// The function every producer takes the versions from (issue 1921): rows for what a build at a group's heads is asked.
|
||||
h := func(c string) string { return strings.Repeat(c, 40) }
|
||||
const (
|
||||
contracts = "git.novox.be/novox/mesh-contracts/go/clients/assignerclient"
|
||||
tools = "git.novox.be/novox/mesh-tools/go/x"
|
||||
)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
latest delivery.PackageVersions
|
||||
heads map[string]string
|
||||
own string
|
||||
want delivery.PackageVersions
|
||||
}{
|
||||
{"none", nil, nil, "novox/mesh-host", nil},
|
||||
{"trunk latest alone", delivery.PackageVersions{tools: h("a")}, nil, "novox/mesh-host", delivery.PackageVersions{tools: h("a")}},
|
||||
{"a member at its head replaces the latest", delivery.PackageVersions{SDKModulePath: h("a")}, map[string]string{SDKRepository: h("b")}, "novox/mesh-host",
|
||||
delivery.PackageVersions{SDKModulePath: h("b")}},
|
||||
{"a member that publishes no package yet is named by its root module", delivery.PackageVersions{SDKModulePath: h("a")},
|
||||
map[string]string{"novox/mesh-contracts": h("c")}, "novox/mesh-host",
|
||||
delivery.PackageVersions{SDKModulePath: h("a"), "git.novox.be/novox/mesh-contracts": h("c")}},
|
||||
{"mesh-sdk is named by its go module before any package of it is known", nil, map[string]string{SDKRepository: h("b")}, "novox/mesh-host",
|
||||
delivery.PackageVersions{SDKModulePath: h("b")}},
|
||||
{"a member whose packages are known gets no root", delivery.PackageVersions{contracts: h("a")},
|
||||
map[string]string{"novox/mesh-contracts": h("c")}, "novox/mesh-host", delivery.PackageVersions{contracts: h("c")}},
|
||||
{"the build's own repository is left out", delivery.PackageVersions{contracts: h("a")},
|
||||
map[string]string{"novox/mesh-contracts": h("c")}, "novox/mesh-contracts", nil},
|
||||
{"a head that is no full commit names nothing", nil, map[string]string{"novox/mesh-contracts": "abc"}, "novox/mesh-host", nil},
|
||||
} {
|
||||
got := grouppackages.Heads(tc.latest, tc.heads, tc.own)
|
||||
if !maps.Equal(got, tc.want) {
|
||||
t.Errorf("%s: got %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type runsAt map[string]string
|
||||
@@ -111,10 +154,16 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
||||
}
|
||||
// Alone, mesh-sdk's tests read the catalogue's trunk, and the verdict names it.
|
||||
b = newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2})
|
||||
const contracts = "novox/mesh-contracts"
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2, contracts: trunkA, DeliveryRepository: trunkA2})
|
||||
if out := b.judge(t, caseOf(t, pullOf(SDKRepository, 7, head1))); out.Beside[catalogue] != trunkA {
|
||||
t.Errorf("the siblings named for the verdict alone = %v, want the catalogue at its trunk %s", out.Beside, trunkA)
|
||||
}
|
||||
// mesh-sdk's tests compare its bootstrapped seats with their contracts files, read beside it: mesh-contracts and
|
||||
// mesh-delivery, at their trunks alone (a group try is placed by the same besideFor, so it lays them out the same way).
|
||||
if got := besideOf(t, b, SDKRepository); got[contracts].Commit != trunkA || got[DeliveryRepository].Commit != trunkA2 {
|
||||
t.Errorf("beside mesh-sdk alone = %+v, want mesh-contracts at %s and mesh-delivery at %s", got, trunkA, trunkA2)
|
||||
}
|
||||
if c := besideOf(t, b, SDKRepository)[catalogue]; c.Commit != trunkA || c.Ref != "" {
|
||||
t.Errorf("the catalogue beside mesh-sdk alone = %+v, want its trunk %s", c, trunkA)
|
||||
}
|
||||
@@ -168,6 +217,18 @@ func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
|
||||
if h := got[nodeEngineRepo]; h.Commit != hostRuns {
|
||||
t.Errorf("the node-engine beside a catalogue change = %+v, want the commit the mesh runs %s", h, hostRuns)
|
||||
}
|
||||
// A group that holds the node-engine is judged against its head, which is what runs after its merge, never the commit
|
||||
// the mesh runs now; the verdict names it in no carry (it is a member, not read at a trunk or a running commit).
|
||||
b = newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, nodeEngineRepo: trunkA2})
|
||||
b.ex.Runs = runsAt{nodeEngineModule: hostRuns}
|
||||
out = b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1), pullOf(nodeEngineRepo, 8, head2)))
|
||||
if h := besideOf(t, b, catalogueRepository)[nodeEngineRepo]; h.Commit != head2 || h.Ref != "refs/pull/8/head" {
|
||||
t.Errorf("the node-engine beside a catalogue change in a group holding it = %+v, want the member's head %s", h, head2)
|
||||
}
|
||||
if out.Beside[nodeEngineRepo] != "" {
|
||||
t.Errorf("the verdict names the member node-engine in its carry: %v", out.Beside)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A12: a mesh-lab head gets a check: with its trunk known it is judged like any repository, its Go fetch finding
|
||||
|
||||
@@ -3,6 +3,7 @@ package check
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -47,6 +48,9 @@ type Reader struct {
|
||||
Runs interface{ CommitOf(module string) string }
|
||||
// HearingState is the checker's state hearing: whether it hears DEAD_LETTERS (issue 734).
|
||||
HearingState StateRead
|
||||
// BuildState is the builder's builds (mesh-delivery-builder.builds): the builds of a running check's candidate that have
|
||||
// not ended, which the check waits on before it composes (ADR 0603 §10); without it none is named.
|
||||
BuildState StateRead
|
||||
// AskState is the checker's state asks: the asks each running check put (AsksState), which checker_ask_again reads.
|
||||
AskState StateRead
|
||||
Now func() time.Time
|
||||
@@ -160,6 +164,7 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
keys = slices.DeleteFunc(keys, IsTryGroup)
|
||||
sort.Strings(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
|
||||
@@ -344,6 +349,10 @@ type Status struct {
|
||||
// Lost are the candidates whose verdict is running while no step of theirs can still run and their newest step ended, by
|
||||
// group, with that step: nothing will judge them (the lost check of 2026-10-28); checker_ask_again asks one again.
|
||||
Lost map[string]string `json:"lost"`
|
||||
// BuildsNotEnded are, of each candidate whose check runs, the builder's builds of its changed modules that are neither
|
||||
// done nor failed, by group: `<module> <state>`, sorted (no key yet is `not begun`). A check waits on them before it
|
||||
// composes (ADR 0603 §10), but for the modules its line names as composed without them; builder_build_again builds a failed one again, and the check asks itself again when it is done.
|
||||
BuildsNotEnded map[string]string `json:"builds_not_ended,omitempty"`
|
||||
// DeadLetters says whether the checker hears the asks the build seats give up on (issue 734): deaf, a try or check
|
||||
// whose ask is given up waits until its bound.
|
||||
DeadLetters DeadLettersHeard `json:"dead_letters"`
|
||||
@@ -436,6 +445,7 @@ func (r Reader) Status() (Status, error) {
|
||||
if err != nil {
|
||||
return st, err
|
||||
}
|
||||
keys = slices.DeleteFunc(keys, IsTryGroup) // a try's candidate is judged by its try, never as a check
|
||||
sort.Strings(keys)
|
||||
total := len(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
@@ -500,6 +510,12 @@ func (r Reader) Status() (Status, error) {
|
||||
if why := read.lostWhy(c); why != "" {
|
||||
st.Lost[c.Group] = why
|
||||
}
|
||||
if words := r.buildsNotEnded(c); words != "" {
|
||||
if st.BuildsNotEnded == nil {
|
||||
st.BuildsNotEnded = map[string]string{}
|
||||
}
|
||||
st.BuildsNotEnded[c.Group] = words
|
||||
}
|
||||
case IsNodesFailure(v):
|
||||
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
||||
case IsUntrunked(v):
|
||||
@@ -663,3 +679,38 @@ func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// buildsNotEnded are the builder's builds of the candidate's changed modules that are neither done nor failed, as
|
||||
// Status.BuildsNotEnded says them; empty when none, or when the builder's builds are not readable here.
|
||||
func (r Reader) buildsNotEnded(c delivery.Candidate) string {
|
||||
if r.BuildState.Get == nil {
|
||||
return ""
|
||||
}
|
||||
var out []string
|
||||
for _, p := range c.Pulls {
|
||||
for module, cm := range p.Fps {
|
||||
if cm.FP2 == "" || cm.Manifest == "" {
|
||||
continue
|
||||
}
|
||||
key, err := delivery.BuilderBuildKey(module, cm.FP2)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
raw, err := r.BuildState.Get(key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
state := "not begun"
|
||||
if len(raw) > 0 {
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil || b.State == delivery.BuildDone || b.State == delivery.BuildFailed {
|
||||
continue
|
||||
}
|
||||
state = b.State
|
||||
}
|
||||
out = append(out, module+" "+state)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return strings.Join(slices.Compact(out), ", ")
|
||||
}
|
||||
|
||||
@@ -147,10 +147,24 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
||||
passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed})
|
||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||
// ADR 0603 §10: feat/y's check runs and waits on the builds of its changed modules: web building, mail not begun; api
|
||||
// is done and named nowhere.
|
||||
fp, man := "src2:"+strings.Repeat("f", 64), strings.Repeat("e", 64)
|
||||
py := pullOf(repoA, 8, head2)
|
||||
py.Modules, py.Fps = []string{"api", "mail", "web"}, map[string]delivery.CandidateModule{"web": {FP2: fp, Manifest: man}, "mail": {FP2: fp, Manifest: man}, "api": {FP2: fp, Manifest: man}}
|
||||
candY := candidateOf(t, "feat/y", py)
|
||||
ykey, _ := delivery.CandidatesKey("feat/y")
|
||||
runningV, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/y", Heads: "0123456789ab", State: delivery.VerdictRunning})
|
||||
builds := fakeState{}
|
||||
for module, state := range map[string]string{"web": delivery.BuildBuilding, "api": delivery.BuildDone} {
|
||||
k, _ := delivery.BuilderBuildKey(module, fp)
|
||||
builds[k], _ = json.Marshal(delivery.BuilderBuild{V: 1, State: state})
|
||||
}
|
||||
rd := Reader{
|
||||
StepStates: map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()},
|
||||
CandidateState: fakeState{ckey: cand}.read(),
|
||||
VerdictState: fakeState{}.read(),
|
||||
CandidateState: fakeState{ckey: cand, ykey: candY}.read(),
|
||||
VerdictState: fakeState{keyOf(t, "feat/y", candY): runningV}.read(),
|
||||
BuildState: builds.read(),
|
||||
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||
HearingState: fakeState{"dead-letters": []byte(`{"heard":false,"streams":["SEAT_NODE_BUILD_GO"],` +
|
||||
`"why":"DEAD_LETTERS is not heard, so an ask a build seat gives up on is waited for until its bound: SEAT_NODE_BUILD_GO: permissions violation",` +
|
||||
@@ -167,6 +181,9 @@ func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
||||
t.Errorf("unjudged = %v", st.Unjudged)
|
||||
}
|
||||
if got := st.BuildsNotEnded; len(got) != 1 || got["feat/y"] != "mail not begun, web building" {
|
||||
t.Errorf("builds not ended = %v, want feat/y's mail not begun and web building", got)
|
||||
}
|
||||
// Issue 734, review round 1: a checker that does not hear DEAD_LETTERS says so in its status, with why.
|
||||
if st.DeadLetters.Heard || !strings.Contains(st.DeadLetters.Why, "permissions violation") {
|
||||
t.Errorf("dead letters = %+v, want not heard, saying why", st.DeadLetters)
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -27,6 +28,8 @@ const (
|
||||
TryTool = "checker_try"
|
||||
TryReadTool = "checker_try_read"
|
||||
TryEndTool = "checker_try_end"
|
||||
// TriesTool lists the tries that wait or run (OpenTries).
|
||||
TriesTool = "checker_tries"
|
||||
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
|
||||
TriesState = "tries"
|
||||
TriesBucket = delivery.CheckerModule + "_" + TriesState
|
||||
@@ -62,18 +65,28 @@ var (
|
||||
|
||||
// Try is the value of a tries key.
|
||||
type Try struct {
|
||||
V int `json:"v"`
|
||||
ID string `json:"id"`
|
||||
V int `json:"v"`
|
||||
ID string `json:"id"`
|
||||
// Repository, Head, Trunk and Branch are the first member's (the try's headline, and all a try of one repository
|
||||
// names); Members are every member of the try, by repository: each an exact commit merged onto a trunk commit (a
|
||||
// group try, issue 1921). A try written before members read as one member.
|
||||
Repository string `json:"repository"`
|
||||
// Head is the commit tried; Trunk the trunk commit it is merged onto, and Branch the trunk's branch when it was asked by name.
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Members []TryMember `json:"members,omitempty"`
|
||||
// Tests is a go test -run pattern: only those tests run, and nothing is kept. Level is unit or integration: only that
|
||||
// level runs. Empty is every test of both levels.
|
||||
Tests string `json:"tests,omitempty"`
|
||||
Level string `json:"level,omitempty"`
|
||||
State string `json:"state"`
|
||||
// Told is true from the moment a whole try's start is said on the checker's stream until its end is said there: kept in the
|
||||
// try's own state, so an end that is written and not said (the checker stopped between) is said when the checker starts.
|
||||
Told bool `json:"told,omitempty"`
|
||||
// Waiting is what the running try waits for now (its group's candidate from the indexer, ADR 0639 §5), empty when it waits for nothing;
|
||||
// try-read gives it, so a try that sits still says why.
|
||||
Waiting string `json:"waiting,omitempty"`
|
||||
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
|
||||
Said string `json:"said,omitempty"`
|
||||
// Took is the time from its start to its end, as Go writes a duration.
|
||||
@@ -83,6 +96,11 @@ type Try struct {
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
// Composition is what the composition on every placed node came to (a whole try, not one narrowed to a level or to
|
||||
// some tests), and Notes the words the check noted on its way (the manifests judged, the module graph, the kept
|
||||
// verdicts taken): the same as a group's pull-request check reports.
|
||||
Composition *delivery.Composition `json:"composition,omitempty"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
|
||||
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
|
||||
Asks []string `json:"asks,omitempty"`
|
||||
@@ -94,6 +112,23 @@ type Try struct {
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
}
|
||||
|
||||
// TryMember is one repository of a try: the exact commit asked, never the branch's head now, and the trunk commit it is
|
||||
// merged onto. Branch is the trunk's branch when the trunk was asked by name.
|
||||
type TryMember struct {
|
||||
Repository string `json:"repository"`
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
}
|
||||
|
||||
// MembersOf are the try's members: its Members, or the one its headline names.
|
||||
func (t Try) MembersOf() []TryMember {
|
||||
if len(t.Members) > 0 {
|
||||
return t.Members
|
||||
}
|
||||
return []TryMember{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk, Branch: t.Branch}}
|
||||
}
|
||||
|
||||
// TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream.
|
||||
const TryAsked = "asked"
|
||||
|
||||
@@ -148,6 +183,17 @@ func (w *tryWriter) heard(askID string, step *delivery.Step) {
|
||||
}
|
||||
}
|
||||
|
||||
// waiting writes what the try waits for now ("" when it waits for nothing).
|
||||
func (w *tryWriter) waiting(why string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped || w.t.Waiting == why {
|
||||
return
|
||||
}
|
||||
w.t.Waiting = why
|
||||
w.write()
|
||||
}
|
||||
|
||||
// write writes the record at the revision written last; a refusal stops the writes.
|
||||
func (w *tryWriter) write() {
|
||||
data, err := encodeTry(w.t)
|
||||
@@ -195,6 +241,7 @@ func DecodeTry(raw []byte) (Try, error) {
|
||||
if !tryKeyOK.MatchString(t.ID) || t.Repository == "" || t.Head == "" || t.Trunk == "" {
|
||||
return Try{}, errors.New("not a try: it names no id, repository, head or trunk")
|
||||
}
|
||||
t.Members = t.MembersOf()
|
||||
return t, nil
|
||||
}
|
||||
|
||||
@@ -206,6 +253,14 @@ func encodeTry(t Try) ([]byte, error) {
|
||||
// TryAsk is what checker_try is given.
|
||||
type TryAsk struct {
|
||||
Repository, Head, Trunk, Tests, Level string
|
||||
// Members are the repositories of a group try, each an exact commit and, optionally, its trunk (a commit, or the
|
||||
// trunk's branch for its head now); the single Repository, Head and Trunk above are one member. Both at once are refused.
|
||||
Members []TryAskMember
|
||||
}
|
||||
|
||||
// TryAskMember is one repository of a group try as it is asked.
|
||||
type TryAskMember struct {
|
||||
Repository, Head, Trunk string
|
||||
}
|
||||
|
||||
// TryStore is the tries state as the tools see it.
|
||||
@@ -221,21 +276,16 @@ func NotChecked(repository string) error {
|
||||
return fmt.Errorf("no level of %s is checked by the checker", repository)
|
||||
}
|
||||
|
||||
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
|
||||
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the
|
||||
// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
|
||||
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
|
||||
// tryMember checks one member as it is asked and resolves its trunk: the gate's trunk head of its repository, a commit
|
||||
// asked, or the trunk's branch (its head now). A repository with no trunk head is no repository the checker checks.
|
||||
func tryMember(trunks map[string]string, a TryAskMember) (TryMember, error) {
|
||||
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
|
||||
head := strings.ToLower(strings.TrimSpace(a.Head))
|
||||
switch {
|
||||
case !tryRepoOK.MatchString(repo) || strings.Contains(repo, ".."):
|
||||
return "", fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
|
||||
return TryMember{}, fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
|
||||
case !tryHeadOK.MatchString(head):
|
||||
return "", fmt.Errorf("the head %q is not a full commit, 40 or 64 hex digits", a.Head)
|
||||
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
|
||||
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
|
||||
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
|
||||
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
|
||||
return TryMember{}, fmt.Errorf("the head %q of %s is not a full commit, 40 or 64 hex digits", a.Head, repo)
|
||||
}
|
||||
trunkHead := ""
|
||||
for r, h := range trunks {
|
||||
@@ -244,20 +294,67 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
||||
}
|
||||
}
|
||||
if trunkHead == "" {
|
||||
return "", NotChecked(repo)
|
||||
return TryMember{}, NotChecked(repo)
|
||||
}
|
||||
t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
|
||||
m := TryMember{Repository: repo, Head: head}
|
||||
switch trunk := strings.TrimSpace(a.Trunk); {
|
||||
case trunk == "":
|
||||
t.Trunk = trunkHead
|
||||
m.Trunk = trunkHead
|
||||
case tryHeadOK.MatchString(strings.ToLower(trunk)):
|
||||
t.Trunk = strings.ToLower(trunk)
|
||||
m.Trunk = strings.ToLower(trunk)
|
||||
case tryTrunkOK.MatchString(trunk) && !strings.Contains(trunk, ".."):
|
||||
// The checker judges pull requests into the trunk alone (ADR 0484 clause 11): a branch named is the trunk's head now.
|
||||
t.Trunk, t.Branch = trunkHead, trunk
|
||||
m.Trunk, m.Branch = trunkHead, trunk
|
||||
default:
|
||||
return "", fmt.Errorf("the trunk %q is neither a full commit nor a branch", a.Trunk)
|
||||
return TryMember{}, fmt.Errorf("the trunk %q of %s is neither a full commit nor a branch", a.Trunk, repo)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// sameMembers says whether two tries ask the same commits onto the same trunk commits.
|
||||
func sameMembers(a, b []TryMember) bool {
|
||||
return slices.EqualFunc(a, b, func(x, y TryMember) bool {
|
||||
return x.Repository == y.Repository && x.Head == y.Head && x.Trunk == y.Trunk
|
||||
})
|
||||
}
|
||||
|
||||
// AskTry writes a new try, queued, and answers its id: a try of the same commits that waits or runs is the answer to a second
|
||||
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. A try is a group of
|
||||
// one or more members (issue 1921), each an exact commit, never the branch's head now: a push after the ask changes nothing
|
||||
// of what is tried. trunks are the gate's trunk heads of every repository (source.main): a repository with none is no
|
||||
// repository the checker checks. A trunk asked by name is its trunk head now.
|
||||
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
|
||||
asked := a.Members
|
||||
if a.Repository != "" || a.Head != "" || a.Trunk != "" {
|
||||
if len(asked) > 0 {
|
||||
return "", errors.New("name the members, or one repository with its head, not both")
|
||||
}
|
||||
asked = []TryAskMember{{Repository: a.Repository, Head: a.Head, Trunk: a.Trunk}}
|
||||
}
|
||||
if len(asked) == 0 {
|
||||
return "", errors.New("name what to try: members (repository and head each), or a repository and its head")
|
||||
}
|
||||
switch {
|
||||
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
|
||||
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
|
||||
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
|
||||
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
|
||||
}
|
||||
var members []TryMember
|
||||
for _, m := range asked {
|
||||
tm, err := tryMember(trunks, m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if slices.ContainsFunc(members, func(o TryMember) bool { return o.Repository == tm.Repository }) {
|
||||
return "", fmt.Errorf("%s is named twice: a try holds one commit of a repository", tm.Repository)
|
||||
}
|
||||
members = append(members, tm)
|
||||
}
|
||||
slices.SortFunc(members, func(x, y TryMember) int { return strings.Compare(x.Repository, y.Repository) })
|
||||
first := members[0]
|
||||
t := Try{Repository: first.Repository, Head: first.Head, Trunk: first.Trunk, Branch: first.Branch, Members: members,
|
||||
Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
|
||||
keys, err := store.Keys()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the tries cannot be listed: %w", err)
|
||||
@@ -291,10 +388,10 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
||||
if !live[i] {
|
||||
continue
|
||||
}
|
||||
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
|
||||
if sameMembers(cur.MembersOf(), t.Members) && cur.Tests == t.Tests && cur.Level == t.Level {
|
||||
return cur.ID, nil
|
||||
}
|
||||
said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)
|
||||
said := fmt.Sprintf("%s (%s, %s)", cur.ID, sayMembers(cur.MembersOf()), cur.State)
|
||||
if cur.HoldsANode() {
|
||||
holding = append(holding, said)
|
||||
} else {
|
||||
@@ -379,12 +476,50 @@ func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// IsTryGroup says a candidates key is the group of a try (the indexer makes the candidate of a group try's previews, for the
|
||||
// manifest validator and the composition): the try judges it, so the checker's check of candidates leaves it alone, writes
|
||||
// no verdict and owes the forge no status for it.
|
||||
func IsTryGroup(key string) bool { return tryKeyOK.MatchString(key) }
|
||||
|
||||
// TryOptions narrow what a try runs; a pull request's check has none.
|
||||
type TryOptions struct {
|
||||
Level string
|
||||
Run string
|
||||
}
|
||||
|
||||
// Whole says the try runs everything a group's pull-request check runs (both levels of every member, the manifests judged by
|
||||
// the manifest validator, the module graph, the composition on every placed node): it is narrowed to neither a level nor
|
||||
// some tests. Such a try judges by the same code as the check, so the two cannot drift (issue 1921).
|
||||
func (o *TryOptions) Whole() bool { return o != nil && o.Level == "" && o.Run == "" }
|
||||
|
||||
// TryCase is the case a try is judged as: one pull of the group per member at its exact commit, with no pull request number
|
||||
// (no ref: the commit is fetched as it is), the group named by the try's key, and the heads hash its candidate is keyed by.
|
||||
func TryCase(key string, t Try, watch func(string, *delivery.Step)) *Case {
|
||||
cs := &Case{Key: key, Group: key, Watch: watch, Trunk: map[string]string{}, Queued: t.Asked, Progress: &Progress{},
|
||||
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
|
||||
var heads []delivery.Head
|
||||
for _, m := range t.MembersOf() {
|
||||
cs.Pulls = append(cs.Pulls, Pull{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
|
||||
cs.Trunk[m.Repository] = m.Trunk
|
||||
heads = append(heads, delivery.Head{Repository: m.Repository, Head: m.Head})
|
||||
}
|
||||
cs.HeadsHash, _ = delivery.HeadsHash(heads)
|
||||
cs.Heads = cs.HeadsHash
|
||||
return cs
|
||||
}
|
||||
|
||||
// compositionMade says whether a composition was made; notMadeWords what it says when it was not.
|
||||
func compositionMade(c *delivery.Composition) bool {
|
||||
return c != nil && c.State != delivery.CompositionNotRun
|
||||
}
|
||||
|
||||
func notMadeWords(c *delivery.Composition) string {
|
||||
if c == nil || c.Words == "" {
|
||||
return "no reason was given"
|
||||
}
|
||||
return c.Words
|
||||
}
|
||||
|
||||
// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check (a check ask, ADR
|
||||
// 0525 §1), and writes its end. It holds no timer: a try starts when its key is written.
|
||||
type Tryer struct {
|
||||
@@ -396,7 +531,13 @@ type Tryer struct {
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
// Say publishes an event of the checker's stream (tryevent.Started, tryevent.Ended) with its body; nil says nothing. The
|
||||
// indexer, which hears them, writes and deletes the candidate of a whole try's group.
|
||||
// It answers the publish's error, nil only when the stream acknowledged it: an end not acknowledged is said again.
|
||||
Say func(ctx context.Context, event string, body tryevent.Event) error
|
||||
|
||||
mu sync.Mutex
|
||||
ending map[string]bool // tries whose end is being said now
|
||||
wg sync.WaitGroup
|
||||
inflight map[string]context.CancelFunc
|
||||
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
|
||||
@@ -436,6 +577,7 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
|
||||
stop()
|
||||
}
|
||||
tr.mu.Unlock()
|
||||
tr.sendEnd(ctx, key) // ended by hand while it ran, or its end was written and not said: the indexer drops its candidate
|
||||
return
|
||||
}
|
||||
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
|
||||
@@ -491,6 +633,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
return
|
||||
}
|
||||
t.State, t.Started = TryRunning, tr.now().UTC()
|
||||
t.Told = (&TryOptions{Level: t.Level, Run: t.Tests}).Whole() // its start is said below; its end is owed from now on
|
||||
data, err := encodeTry(t)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -502,12 +645,15 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
return
|
||||
}
|
||||
progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev}
|
||||
cs := &Case{Key: key, Group: key, Watch: progress.heard,
|
||||
Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}},
|
||||
Trunk: map[string]string{t.Repository: t.Trunk},
|
||||
Queued: t.Asked,
|
||||
Progress: &Progress{},
|
||||
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
|
||||
if t.Told {
|
||||
started := tryevent.Event{ID: key}
|
||||
for _, m := range t.MembersOf() {
|
||||
started.Members = append(started.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
tr.say(ctx, tryevent.Started, started)
|
||||
}
|
||||
cs := TryCase(key, t, progress.heard)
|
||||
cs.Wait = progress.waiting
|
||||
judging := ctx
|
||||
if tr.NotTaken > 0 {
|
||||
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
|
||||
@@ -517,12 +663,18 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
|
||||
}
|
||||
t, rev = progress.end()
|
||||
t.Waiting = ""
|
||||
v := out.Verdict
|
||||
t.Ended = tr.now().UTC()
|
||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
|
||||
t.Composition = v.Composition
|
||||
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
||||
for _, n := range cs.Notes() {
|
||||
if len(cs.Pulls) > 1 {
|
||||
t.Said = sayMembers(t.MembersOf()) + ": " + t.Said
|
||||
}
|
||||
t.Notes = cs.Notes()
|
||||
for _, n := range t.Notes {
|
||||
t.Said += " | " + n
|
||||
}
|
||||
switch {
|
||||
@@ -532,6 +684,16 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
if IsLostAsk(v) || IsNotTaken(v) {
|
||||
t.Failed = []string{v.Failed.Words}
|
||||
}
|
||||
case v.State == delivery.VerdictPass && cs.Try.Whole() && compositionMade(v.Composition) && strings.Contains(v.Composition.Words, delivery.ComposeNoAnswer):
|
||||
// A node that gave no answer passes a group's check with a warning; a try that is to say the group composes says it
|
||||
// did not, naming the node (ADR 0637 §3).
|
||||
t.State = TryError
|
||||
t.Failed = []string{"the composition was not made on every placed node: " + v.Composition.Words}
|
||||
case v.State == delivery.VerdictPass && cs.Try.Whole() && !compositionMade(v.Composition):
|
||||
// A whole try that judged the tests and not the manifests and the composition has not tried what its group's check
|
||||
// judges: it is no pass, and it says why (failure must be loud). It is an error, not the change's failure.
|
||||
t.State = TryError
|
||||
t.Failed = []string{"the composition was not made: " + notMadeWords(v.Composition)}
|
||||
case v.State == delivery.VerdictPass:
|
||||
t.State = TryPass
|
||||
default:
|
||||
@@ -549,6 +711,97 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
return
|
||||
}
|
||||
tr.logf("%s: %s", key, t.State)
|
||||
tr.sendEnd(ctx, key)
|
||||
}
|
||||
|
||||
// sendEnd says the end of an ended try whose start was said (Told) with the state it ended in, then clears Told in the try's
|
||||
// own state; a checker that stopped between the two says it again at its start. Every end is said so: pass, fail, error and
|
||||
// by hand.
|
||||
func (tr *Tryer) sendEnd(ctx context.Context, key string) {
|
||||
tr.mu.Lock()
|
||||
if tr.ending[key] {
|
||||
tr.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if tr.ending == nil {
|
||||
tr.ending = map[string]bool{}
|
||||
}
|
||||
tr.ending[key] = true
|
||||
tr.mu.Unlock()
|
||||
defer func() { tr.mu.Lock(); delete(tr.ending, key); tr.mu.Unlock() }()
|
||||
raw, rev, found, err := tr.Tries.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
t, err := DecodeTry(raw)
|
||||
if err != nil || !t.Over() || !t.Told {
|
||||
return
|
||||
}
|
||||
if tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) != nil {
|
||||
return // not acknowledged: Told stays, and the next Put or Reconsider says the end again
|
||||
}
|
||||
t.Told = false
|
||||
data, err := encodeTry(t)
|
||||
if err == nil {
|
||||
_, err = tr.Tries.Update(ctx, key, data, rev)
|
||||
}
|
||||
if err != nil && !errors.Is(err, ErrConflict) {
|
||||
tr.logf("%s: its end was said and Told could not be cleared: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// say publishes a try event; a failure is said in the log (the indexer finds the open tries again at its start).
|
||||
func (tr *Tryer) say(ctx context.Context, event string, body tryevent.Event) error {
|
||||
if tr.Say == nil {
|
||||
return nil
|
||||
}
|
||||
err := tr.Say(ctx, event, body)
|
||||
if err != nil {
|
||||
tr.logf("%s of %s could not be said: %v", event, body.ID, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// OpenTries are every try that waits or runs now: what checker_tries answers, so a part that missed an event (the indexer at
|
||||
// its start) finds them without reading the checker's state. All the keys are read (an old open try is as open as a new one), in
|
||||
// parallel.
|
||||
func OpenTries(store TryStore) ([]tryevent.Open, error) {
|
||||
keys, err := store.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the tries cannot be listed: %w", err)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(keys[i]) != len(keys[j]) {
|
||||
return len(keys[i]) > len(keys[j])
|
||||
}
|
||||
return keys[i] > keys[j]
|
||||
})
|
||||
found := make([]*tryevent.Open, len(keys))
|
||||
if err := parallel(len(keys), func(i int) error {
|
||||
raw, err := store.Get(keys[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
t, err := DecodeTry(raw)
|
||||
if err != nil || t.Over() {
|
||||
return nil
|
||||
}
|
||||
o := tryevent.Open{ID: t.ID, State: t.State, Whole: (&TryOptions{Level: t.Level, Run: t.Tests}).Whole()}
|
||||
for _, m := range t.MembersOf() {
|
||||
o.Members = append(o.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
found[i] = &o
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []tryevent.Open{} // none open is an empty list, never null: a reader tells it from no answer (issue 1921)
|
||||
for _, o := range found {
|
||||
if o != nil {
|
||||
out = append(out, *o)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A
|
||||
@@ -579,6 +832,15 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O
|
||||
return out, true
|
||||
}
|
||||
|
||||
// sayMembers names a try's members, each repository at its short commit.
|
||||
func sayMembers(ms []TryMember) string {
|
||||
var out []string
|
||||
for _, m := range ms {
|
||||
out = append(out, fmt.Sprintf("%s@%.12s", m.Repository, m.Head))
|
||||
}
|
||||
return strings.Join(out, " + ")
|
||||
}
|
||||
|
||||
// orNone joins the tries said, or says none.
|
||||
func orNone(said []string) string {
|
||||
if len(said) == 0 {
|
||||
|
||||
@@ -3,11 +3,14 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -70,6 +73,25 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR
|
||||
}
|
||||
// Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or
|
||||
// their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts.
|
||||
// A group try is the same try whatever the order of its members; the commits asked are the commits tried.
|
||||
group := TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2}, {Repository: repoA, Head: head1}}}
|
||||
gid, err := AskTry(store, tryTrunks, group, at.Add(time.Second))
|
||||
if err != nil || gid == id {
|
||||
t.Fatalf("the group try = %q, %v, want a try of its own", gid, err)
|
||||
}
|
||||
group.Members[0], group.Members[1] = group.Members[1], group.Members[0]
|
||||
if again, err := AskTry(store, tryTrunks, group, at.Add(time.Minute)); err != nil || again != gid {
|
||||
t.Errorf("the group asked in another order answered %q, %v, want %q", again, err, gid)
|
||||
}
|
||||
if g := readTry(t, kv, gid); len(g.Members) != 2 || g.Members[0].Repository != repoA || g.Members[0].Head != head1 || g.Members[1].Trunk != trunkS {
|
||||
t.Errorf("group try = %+v, want its members by repository at the commits asked", g.Members)
|
||||
}
|
||||
if open, err := OpenTries(store); err != nil || len(open) != 2 || !open[0].Whole || len(open[1].Members) != 2 && len(open[0].Members) != 2 {
|
||||
t.Errorf("open tries = %+v, %v, want the two queued", open, err)
|
||||
}
|
||||
if !IsTryGroup(gid) || IsTryGroup("feat/x") || IsTryGroup("fix/1921-try-1") {
|
||||
t.Errorf("which candidates keys are a try's group: %s is, a branch is not", gid)
|
||||
}
|
||||
onNode := func(i int) Try {
|
||||
return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning,
|
||||
Steps: []TryStep{{Ask: "a", State: delivery.StepRunning}}}
|
||||
@@ -112,59 +134,154 @@ func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agr
|
||||
}
|
||||
}
|
||||
for _, bad := range []TryAsk{{Repository: repoA, Head: "abc"}, {Repository: repoA, Head: head1, Level: "e2e"},
|
||||
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"}} {
|
||||
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"},
|
||||
// A group try (issue 1921): members or one repository, never both or none; one commit per repository; every member checked.
|
||||
{},
|
||||
{Repository: repoA, Head: head1, Members: []TryAskMember{{Repository: repoSDK, Head: head2}}},
|
||||
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoA, Head: head2}}},
|
||||
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoSDK, Head: "main"}}},
|
||||
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: "novox/no-trunk", Head: head2}}}} {
|
||||
if _, err := AskTry(store, tryTrunks, bad, time.Now()); err == nil {
|
||||
t.Errorf("%+v was taken", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5
|
||||
// fakeTryCandidates answers the indexer's candidate of a try's group from memory, recording the keys asked.
|
||||
type fakeTryCandidates struct {
|
||||
cand delivery.Candidate
|
||||
err error
|
||||
asked []string
|
||||
// during is called while the try waits for the candidate.
|
||||
during func()
|
||||
}
|
||||
|
||||
func (f *fakeTryCandidates) TryCandidate(_ context.Context, group, heads string, _ time.Duration) (delivery.Candidate, error) {
|
||||
if f.during != nil {
|
||||
f.during()
|
||||
}
|
||||
f.asked = append(f.asked, group+"."+heads)
|
||||
return f.cand, f.err
|
||||
}
|
||||
|
||||
// wholeBench makes the bench judge a whole try as the checker does: the composition on every placed node and the
|
||||
// indexer's candidate of the try's group.
|
||||
func wholeBench(b *bench, cand delivery.Candidate) (*fakeCompositions, *fakeTryCandidates) {
|
||||
comps := &fakeCompositions{v: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14", "shanks"}, Took: 40}}
|
||||
cands := &fakeTryCandidates{cand: cand}
|
||||
b.ex.Compositions, b.ex.Candidates = comps, cands
|
||||
return comps, cands
|
||||
}
|
||||
|
||||
// A group try (issue 1921): several repositories each at the exact commit asked, judged by the code of a group's pull-request
|
||||
// check; the two cannot drift, and a later check of the same heads keeps what the try passed.
|
||||
func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5, issue 1921
|
||||
ctx := context.Background()
|
||||
kv := newMemKV()
|
||||
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: trunkA}, time.Now())
|
||||
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2, Trunk: trunkS}, {Repository: repoA, Head: head1, Trunk: trunkA}}}, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
comps, cands := wholeBench(b, delivery.Candidate{FPSet: "fps-of-the-try"})
|
||||
written := &writtenTries{memKV: kv}
|
||||
tr := &Tryer{Tries: written, Judge: b.ex}
|
||||
var waiting string
|
||||
cands.during = func() { waiting = readTry(t, kv, id).Waiting } // ADR 0639 §5: the stored try says what it waits for
|
||||
var said []string
|
||||
tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
|
||||
said = append(said, fmt.Sprintf("%s %s %d %s", event, e.ID, len(e.Members), e.State))
|
||||
return nil
|
||||
}}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
// The indexer hears the start with the members at their commits, and the end (it writes and deletes the group's candidate).
|
||||
if len(said) != 2 || said[0] != "try.started "+id+" 2 " || said[1] != "try.ended "+id+" 0 "+TryPass {
|
||||
t.Errorf("events = %v, want the start with 2 members and the end", said)
|
||||
}
|
||||
// What the indexer reads: tryevent.Decode accepts the events as said and refuses a body that is no try's.
|
||||
for _, row := range []struct {
|
||||
name, event, body string
|
||||
ok bool
|
||||
}{
|
||||
{"a start with its members", tryevent.Started, `{"id":"` + id + `","members":[{"repository":"` + repoA + `","head":"` + head1 + `","trunk":"` + trunkA + `"}]}`, true},
|
||||
{"an end with its state", tryevent.Ended, `{"id":"` + id + `","state":"pass"}`, true},
|
||||
{"an id that is no try's", tryevent.Ended, `{"id":"pull-7","state":"pass"}`, false},
|
||||
{"a start with no member", tryevent.Started, `{"id":"` + id + `"}`, false},
|
||||
{"a body that is no JSON", tryevent.Ended, `try-7`, false},
|
||||
} {
|
||||
if e, err := tryevent.Decode(row.event, []byte(row.body)); (err == nil) != row.ok || (row.ok && e.ID != id) {
|
||||
t.Errorf("Decode of %s = %+v, %v, want ok=%v", row.name, e, err, row.ok)
|
||||
}
|
||||
}
|
||||
got := readTry(t, kv, id)
|
||||
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
|
||||
if !strings.Contains(waiting, "candidate") || got.Waiting != "" {
|
||||
t.Errorf("waiting while the candidate was awaited = %q, after the end %q: want the reason, then none", waiting, got.Waiting)
|
||||
}
|
||||
if got.Told { // the end was said: nothing is owed
|
||||
t.Errorf("the ended try still owes its end: %+v", got)
|
||||
}
|
||||
if open, err := OpenTries(memTries{kv}); err != nil || len(open) != 0 {
|
||||
t.Errorf("open tries after the end = %v, %v", open, err)
|
||||
} else if raw, _ := json.Marshal(open); string(raw) != "[]" { // issue 1921: none open is [] on the wire, never null
|
||||
t.Errorf("none open is answered %s, want []", raw)
|
||||
}
|
||||
if got.State != TryPass || got.Took == "" || len(got.Members) != 2 || got.Members[0].Repository != repoA || got.Members[1].Head != head2 ||
|
||||
got.Said == "" || got.Levels[delivery.LevelUnit].Run+got.Levels[delivery.LevelUnit].Kept != 4 { // the bench gives both members the same digests
|
||||
t.Fatalf("try = %+v", got)
|
||||
}
|
||||
// Issue 1890: while it runs, the try names its ask as soon as it is put, and then where its step stands; the end
|
||||
// keeps each ask's node and state.
|
||||
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked ||
|
||||
written.running[2].Steps[0].Node != "shanks" {
|
||||
t.Errorf("the running try's writes = %+v, want its ask named while it ran, then its node", written.running)
|
||||
if got.Composition == nil || got.Composition.State != delivery.CompositionPass || len(got.Machines) == 0 {
|
||||
t.Errorf("the try's composition = %+v, want the composition on every placed node, as a check's", got.Composition)
|
||||
}
|
||||
if len(got.Steps) != 1 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed {
|
||||
t.Errorf("the ended try's steps = %+v", got.Steps)
|
||||
// The same key the indexer's candidate of the group has: the try's id and the heads of its members.
|
||||
if len(comps.asked) != 1 || comps.asked[0] != id || len(cands.asked) != 1 || !strings.HasPrefix(cands.asked[0], id+".") {
|
||||
t.Errorf("composed %v, candidate asked %v, want the try's key %s", comps.asked, cands.asked, id)
|
||||
}
|
||||
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
|
||||
if len(got.Asks) != 1 {
|
||||
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
|
||||
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
|
||||
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
|
||||
// Issue 1890: while it runs, the try names its asks as soon as they are put, and then where each step stands.
|
||||
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked {
|
||||
t.Errorf("the running try's writes = %+v, want its ask named while it ran", written.running)
|
||||
}
|
||||
if len(got.Steps) != 2 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed || len(got.Asks) != 2 {
|
||||
t.Errorf("the ended try's steps = %+v, asks %v", got.Steps, got.Asks)
|
||||
}
|
||||
for _, a := range got.Asks { // the delivery view counts their steps as a try's (build_usage)
|
||||
if _, err := delivery.AskStepKey("shanks", a); err != nil {
|
||||
t.Errorf("the try names %q, which is no ask id: %v", a, err)
|
||||
}
|
||||
}
|
||||
// Each member is fetched at the exact commit asked, merged onto its own trunk, with no pull request's ref.
|
||||
fetch := b.steps.tool("fetch")
|
||||
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
|
||||
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
|
||||
if len(fetch) != 2 {
|
||||
t.Fatalf("fetch = %+v, want one per member", fetch)
|
||||
}
|
||||
if keys, _ := b.prev.Keys(ctx); len(keys) != 0 {
|
||||
t.Errorf("a try wrote the previews %v", keys)
|
||||
for _, f := range fetch {
|
||||
want := map[string][2]string{repoA: {head1, trunkA}, repoSDK: {head2, trunkS}}[f.args["repository"].(string)]
|
||||
if f.args["commit"] != want[0] || f.args["base"] != want[1] || f.args["ref"] != "" {
|
||||
t.Errorf("fetch = %+v, want the exact commit %s merged onto %s", f.args, want[0], want[1])
|
||||
}
|
||||
}
|
||||
// A whole try writes the merge previews the indexer makes its candidate from; it writes no status and asks no Approve.
|
||||
if keys, _ := b.prev.Keys(ctx); len(keys) != 2 {
|
||||
t.Errorf("a whole try wrote the previews %v, want one per member", keys)
|
||||
}
|
||||
// An ended try is not run again.
|
||||
raw, _, _, _ = kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
if n := len(b.steps.tool("fetch")); n != 1 {
|
||||
if n := len(b.steps.tool("fetch")); n != 2 {
|
||||
t.Errorf("an ended try was fetched again: %d fetches", n)
|
||||
}
|
||||
// The kept verdicts: what the try passed is kept under the digests of the merged trees, so the pull-request check of
|
||||
// the same head on the same trunk runs none of it again (and a push of other code runs only what changed).
|
||||
before := len(b.steps.tool("test"))
|
||||
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
if out.Verdict.State != delivery.VerdictPass || out.Verdict.Levels[delivery.LevelUnit].Kept == 0 || out.Verdict.Levels[delivery.LevelUnit].Run != 0 ||
|
||||
out.Verdict.Levels[delivery.LevelIntegration].Run != 0 {
|
||||
t.Errorf("the check of the tried head = %+v, want everything the try passed kept and nothing run", out.Verdict.Levels)
|
||||
}
|
||||
if after := len(b.steps.tool("test")); after != before {
|
||||
t.Errorf("the check of the tried head ran %d test steps, want none: the try's passes are kept", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
|
||||
@@ -175,10 +292,16 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex}
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(context.Context, string, tryevent.Event) error {
|
||||
t.Error("a try of one level and some tests said an event: it has no candidate")
|
||||
return nil
|
||||
}}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
if open, _ := OpenTries(memTries{kv}); len(open) != 0 {
|
||||
t.Errorf("open = %v", open)
|
||||
}
|
||||
tests := b.steps.tool("test")
|
||||
if len(tests) != 1 || tests[0].args["level"] != delivery.LevelIntegration || tests[0].args["run"] != "TestBus|TestStore" {
|
||||
t.Fatalf("tests = %+v, want the integration level alone with the run pattern", tests)
|
||||
@@ -192,6 +315,24 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
|
||||
if got := readTry(t, kv, id); got.State != TryPass || !strings.Contains(got.Said, "only the tests TestBus|TestStore") {
|
||||
t.Errorf("try = %+v", got)
|
||||
}
|
||||
t.Run("a try of one level keeps only that level", func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: delivery.LevelUnit}, time.Now())
|
||||
b := newBench(t)
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
keys, _ := b.kept.Keys(ctx)
|
||||
if got := readTry(t, kv, id); got.State != TryPass || len(keys) == 0 {
|
||||
t.Fatalf("try = %+v, kept %v", got, keys)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, delivery.LevelUnit+".") {
|
||||
t.Errorf("a try of the unit level kept %s", k)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) {
|
||||
@@ -208,26 +349,135 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T)
|
||||
!strings.Contains(got.Failed[1], "integration: fixture/failing/bad.TestThreeIsFour") { // both levels ran (the operator, 2026-11-02)
|
||||
t.Errorf("try = %+v", got)
|
||||
}
|
||||
// A whole try that could not judge the manifests and the composition is no pass; a failed composition is named beside
|
||||
// the failed tests (the group's check does as much), and a candidate of other heads is not the try's.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
cand *fakeTryCandidates
|
||||
comp delivery.CompositionVerdict
|
||||
state, said string
|
||||
}{
|
||||
{name: "the indexer wrote no candidate", cand: &fakeTryCandidates{err: errors.New("the indexer wrote no candidate within 5m0s")}, state: TryError, said: "the indexer wrote no candidate within 5m0s"},
|
||||
{name: "a node that gave no composition answer", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x", Warnings: []string{"g14: " + delivery.ComposeNoAnswer}},
|
||||
state: TryError, said: "g14: " + delivery.ComposeNoAnswer},
|
||||
{name: "a composition refused on a node", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "not composed on g14: web: needs a state mail does not keep"},
|
||||
state: TryFail, said: "not composed on g14: web: needs a state mail does not keep"},
|
||||
} {
|
||||
kv := newMemKV()
|
||||
b := newBench(t)
|
||||
comps, _ := wholeBench(b, delivery.Candidate{})
|
||||
b.ex.Candidates, comps.v = row.cand, row.comp
|
||||
if row.comp.State == "" {
|
||||
comps.v.State = delivery.CompositionPass
|
||||
}
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
var ends []string
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
|
||||
if event == tryevent.Ended {
|
||||
ends = append(ends, e.State)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
if len(ends) != 1 || ends[0] != row.state { // every end says its state: error as well as fail
|
||||
t.Errorf("%s: ends said = %v, want once %s", row.name, ends, row.state)
|
||||
}
|
||||
if got := readTry(t, kv, id); got.State != row.state || !strings.Contains(strings.Join(got.Failed, " | ")+got.Said, row.said) {
|
||||
t.Errorf("%s: try = %s, failed %v, said %q, want %s naming %q", row.name, got.State, got.Failed, got.Said, row.state, row.said)
|
||||
}
|
||||
}
|
||||
// No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4).
|
||||
}
|
||||
|
||||
func TestATryLeftRunningIsTakenOver(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
// Left running by a checker that stopped.
|
||||
raw, rev, _, _ := kv.Get(ctx, id)
|
||||
cur, _ := DecodeTry(raw)
|
||||
cur.State = TryRunning
|
||||
data, _ := encodeTry(cur)
|
||||
if _, err := kv.Update(ctx, id, data, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex}
|
||||
tr.Reconsider(ctx)
|
||||
tr.Wait()
|
||||
if got := readTry(t, kv, id); got.State != TryPass {
|
||||
t.Errorf("the try left running = %+v, want taken over and passed", got)
|
||||
// Whether the start was said is in the try's own state (Told), so a checker that starts later says the end of a try it did
|
||||
// not start, whatever way the try ended.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
told bool
|
||||
failFirst bool // the stream does not acknowledge the first end said
|
||||
after func(t *testing.T, kv *memKV, id string)
|
||||
ends []string
|
||||
}{
|
||||
{name: "the first publish of the end is not acknowledged", told: true, failFirst: true, ends: []string{TryPass}},
|
||||
{name: "left running after its start was said", told: true, ends: []string{TryPass}},
|
||||
{name: "left running with no start said", told: false, ends: []string{TryPass}},
|
||||
{name: "ended by hand after the start was said", told: true, ends: []string{TryError}, after: func(t *testing.T, kv *memKV, id string) {
|
||||
if _, err := EndTry(endable{kv}, id, "stuck", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
// Left running by a checker that stopped.
|
||||
raw, rev, _, _ := kv.Get(ctx, id)
|
||||
cur, _ := DecodeTry(raw)
|
||||
cur.State, cur.Told = TryRunning, row.told
|
||||
data, _ := encodeTry(cur)
|
||||
if _, err := kv.Update(ctx, id, data, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if row.after != nil {
|
||||
row.after(t, kv, id)
|
||||
}
|
||||
b := newBench(t)
|
||||
wholeBench(b, delivery.Candidate{})
|
||||
var ends []string
|
||||
failed := false
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
|
||||
if event == tryevent.Ended {
|
||||
if row.failFirst && !failed {
|
||||
failed = true
|
||||
return errors.New("the stream did not acknowledge")
|
||||
}
|
||||
ends = append(ends, e.State)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
tr.Reconsider(ctx)
|
||||
tr.Wait()
|
||||
if row.failFirst && !readTry(t, kv, id).Told {
|
||||
t.Errorf("an end the stream did not acknowledge cleared Told: it is never said again")
|
||||
}
|
||||
tr.Reconsider(ctx) // said once: the second look finds nothing owed
|
||||
tr.Wait()
|
||||
if row.after == nil {
|
||||
if got := readTry(t, kv, id); got.State != TryPass {
|
||||
t.Errorf("the try left running = %+v, want taken over and passed", got)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(ends, row.ends) || readTry(t, kv, id).Told {
|
||||
t.Errorf("ends said = %v, want %v, and nothing owed after", ends, row.ends)
|
||||
}
|
||||
})
|
||||
}
|
||||
// An ended try is kept TryKeep and removed by the next Reconsider after.
|
||||
t.Run("an ended try is kept seven days", func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
raw, rev, _, _ := kv.Get(ctx, id)
|
||||
cur, _ := DecodeTry(raw)
|
||||
ended := time.Now()
|
||||
cur.State, cur.Ended = TryPass, ended
|
||||
data, _ := encodeTry(cur)
|
||||
if _, err := kv.Update(ctx, id, data, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := ended.Add(TryKeep - time.Hour)
|
||||
tr := &Tryer{Tries: kv, Now: func() time.Time { return now }}
|
||||
tr.Reconsider(ctx)
|
||||
if _, _, found, _ := kv.Get(ctx, id); !found {
|
||||
t.Fatal("the try was removed before TryKeep")
|
||||
}
|
||||
now = ended.Add(TryKeep + time.Hour)
|
||||
tr.Reconsider(ctx)
|
||||
if _, _, found, _ := kv.Get(ctx, id); found {
|
||||
t.Error("the try was kept past TryKeep")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -130,14 +130,40 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
if m.SecretsOwner != "mesh-delivery-checker" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3).
|
||||
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**"}}}; !reflect.DeepEqual(m.Streams, want) {
|
||||
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3). A whole try says try.started with its members and try.ended, so the indexer writes and deletes the candidate of its group (issue 1921).
|
||||
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}}; !reflect.DeepEqual(m.Streams, want) {
|
||||
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
||||
}
|
||||
// Only the checker's account publishes try.* on its stream checks: the stream is declared by its owner alone, and no other
|
||||
// module of the repository declares a stream carrying a try event (the indexer and the others only consume them).
|
||||
siblings, err := filepath.Glob("../*/module.json")
|
||||
if err != nil || len(siblings) < 2 {
|
||||
t.Fatalf("the sibling manifests = %v, %v", siblings, err)
|
||||
}
|
||||
for _, f := range siblings {
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var other struct {
|
||||
Module string `json:"module"`
|
||||
Streams []stream `json:"streams"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &other); err != nil {
|
||||
t.Fatalf("%s: %v", f, err)
|
||||
}
|
||||
for _, st := range other.Streams {
|
||||
for _, e := range st.Events {
|
||||
if strings.HasPrefix(e, "try.") && other.Module != m.Module {
|
||||
t.Errorf("%s declares the stream %s with %s: only the checker's account publishes try events", other.Module, st.Name, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
||||
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check
|
||||
// is asked again by hand (checker_ask_again).
|
||||
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code.
|
||||
@@ -174,13 +200,16 @@ func TestManifestProcess(t *testing.T) { // 1678/AC1
|
||||
if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_CHECKER_SETTINGS"] != "${dir:state}/settings.json" {
|
||||
t.Errorf("env = %v, want the broker file and the settings file", p["env"])
|
||||
}
|
||||
// Its settings hold its answer bound alone (ADR 0593 §7: compose-answer-bound, 60 s, the checker's), never a capacity
|
||||
// of its own (ADR 0525 §4).
|
||||
// Its settings hold its answer bound (ADR 0593 §7: compose-answer-bound, 60 s, the checker's) and its wait for the
|
||||
// candidate's builds (ADR 0603 §10: build-wait-bound, 1800 s), never a capacity of its own (ADR 0525 §4).
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "settings" && !reflect.DeepEqual(r["takes-config"], []any{"compose-answer-bound"}) {
|
||||
t.Errorf("the settings resource %v takes more than compose-answer-bound", r)
|
||||
if r["id"] == "settings" && !reflect.DeepEqual(r["takes-config"], []any{"compose-answer-bound", "build-wait-bound"}) {
|
||||
t.Errorf("the settings resource %v takes other than compose-answer-bound and build-wait-bound", r)
|
||||
}
|
||||
}
|
||||
if c, ok := m.Config["build-wait-bound"]; !ok || c.Default != float64(1800) || c.Trusted {
|
||||
t.Errorf("build-wait-bound = %+v, want 1800 by default, not trusted", c)
|
||||
}
|
||||
if c, ok := m.Config["compose-answer-bound"]; !ok || c.Default != float64(60) || c.Trusted {
|
||||
t.Errorf("compose-answer-bound = %+v, want 60 by default, not trusted", c)
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"mesh-build-node.toolchain.reported"
|
||||
],
|
||||
"streams": [
|
||||
{ "name": "checks", "events": ["preview.**", "owed.**"] }
|
||||
{ "name": "checks", "events": ["preview.**", "owed.**", "try.**"] }
|
||||
],
|
||||
"state": [
|
||||
{ "name": "asks", "history": 1 },
|
||||
@@ -56,6 +56,7 @@
|
||||
"checker_previews",
|
||||
"checker_pull",
|
||||
"checker_status",
|
||||
"checker_tries",
|
||||
"checker_try",
|
||||
"checker_try_end",
|
||||
"checker_try_read",
|
||||
@@ -81,6 +82,14 @@
|
||||
"maximum": 600,
|
||||
"trusted": false,
|
||||
"default": 60
|
||||
},
|
||||
"build-wait-bound": {
|
||||
"type": "number",
|
||||
"description": "seconds a check waits for the builder's builds of its candidate to end before it composes with them (novox/hq ADR 0603 §10); a build failed fails the check, naming it, and a build not ended by then is named in the verdict and the check composes without it.",
|
||||
"minimum": 60,
|
||||
"maximum": 7200,
|
||||
"trusted": false,
|
||||
"default": 1800
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
@@ -106,7 +115,8 @@
|
||||
"mode": "0600",
|
||||
"owner": "mesh-delivery-checker",
|
||||
"takes-config": [
|
||||
"compose-answer-bound"
|
||||
"compose-answer-bound",
|
||||
"build-wait-bound"
|
||||
],
|
||||
"merge": "json",
|
||||
"template": "files/settings.json"
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
// Command mesh-delivery-forge-bridge-tools serves the forge bridge's tools (novox/hq ADR 0633 §4, issue 1585): the runtime
|
||||
// launches it for a call. bridge_groups reads the delivery groups the gate holds and what the bridge kept of their ends;
|
||||
// bridge_group_end_again writes an ask into the bridge's state end-asks and waits for the bridge's answer there, since the
|
||||
// bridge is the other program and serves no request subject (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
"mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// Tool names.
|
||||
const (
|
||||
GroupsTool = "bridge_groups"
|
||||
EndAgainTool = "bridge_group_end_again"
|
||||
// MergedTool answers the merged message of a merge on request (issue 1585).
|
||||
MergedTool = "bridge_merged_message"
|
||||
)
|
||||
|
||||
// asker writes an end-asks value and waits for the bridge's answer to it.
|
||||
type asker func(key string, value []byte) (groups.EndAsk, error)
|
||||
|
||||
// Tools are the bridge's tools: groupsRead reads the groups the gate holds with what the bridge kept of them.
|
||||
func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string) (json.RawMessage, error)) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: MergedTool,
|
||||
Description: "The merged message of one merge, asked again (issue 1585): the message the bridge wrote on its stream FORGE when the " +
|
||||
"pull request was merged into the trunk, built the same way from the forge (the merge commit's trees and the manifests " +
|
||||
"and lock files of its module directories) and from the gate's merges state (the pull request's number, head and " +
|
||||
"group). For the indexer, which asks it when it takes in a held merge it does not hold in memory (a restart, a new " +
|
||||
"build), and the ask-again a lost message lacked. About 0.5 MB for a repository of many modules. Answers {message} " +
|
||||
"with the forge message whole, or why it cannot: the gate holds no merge with that commit, or the forge did not answer.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "merge"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "the repository, owner/name, as the forge names it"},
|
||||
"merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
raw, err := merged(str(args, "repository"), str(args, "merge"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"message": raw}, nil
|
||||
}},
|
||||
{Name: GroupsTool,
|
||||
Description: "The delivery groups the gate holds (more than one member), each with its members' merge state and whether each " +
|
||||
"merged message is written on the forge stream, and whether the group's end (forge.group.merged) is written: ended, " +
|
||||
"or open with why (gathered, or merged and its end not written yet). An open group with every member merged is " +
|
||||
"what bridge_group_end_again says the end of. Each group names its last end ask (who, why, when) and the bridge's " +
|
||||
"answer to it, from the state end-asks. A group's name narrows it to that group; open lists only open ones.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "only this group, by the name the gate keys it"},
|
||||
"open": map[string]any{"type": "boolean", "description": "only the groups whose end is not written"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
rows, unread, err := groupsRead()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
only, _ := args["open"].(bool)
|
||||
group := str(args, "group")
|
||||
kept := rows[:0]
|
||||
for _, r := range rows {
|
||||
if (group != "" && r.Group != group) || (only && !r.Open) {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, r)
|
||||
}
|
||||
return map[string]any{"groups": kept, "unread": unread}, nil
|
||||
}},
|
||||
{Name: EndAgainTool,
|
||||
Description: "Say a delivery group's end (forge.group.merged) again, by hand, for a group whose end the indexer did not take " +
|
||||
"in (a group bridge_groups names as merged with its end not written, or one the indexer waits for). The bridge " +
|
||||
"writes it again from the gate's merges, with the time appended to the message id so the stream does not drop it as " +
|
||||
"a duplicate; the indexer takes an end for a group it already took in without changing anything. Refused while " +
|
||||
"a member is not merged or its merged message is not written. Answers whether it was written.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the group, by the name the gate keys it"},
|
||||
"why": map[string]any{"type": "string", "description": "why its end is said again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := groups.NewEndAsk(str(args, "group"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"group": a.Group, "written": a.Written, "answer": a.Answer, "by": a.By, "why": a.Why}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// MergedOf is the merged message of the merge of repository at commit merge, as the bridge writes it: the pull request's
|
||||
// number, head and group from the gate's merges (raw, by key), the rest from the forge through trees and files
|
||||
// (bridge.MergedMessage), and, for a member of a delivery group, the group and its members as the process sets them.
|
||||
func MergedOf(raws map[string][]byte, repository, merge string, trees bridge.Trees, files bridge.Files) (json.RawMessage, error) {
|
||||
owner, repo, ok := strings.Cut(repository, "/")
|
||||
if !ok || owner == "" || repo == "" {
|
||||
return nil, fmt.Errorf("%q is not owner/name", repository)
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(raws)) {
|
||||
m, err := delivery.DecodeMerge(raws[key])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool {
|
||||
return delivery.SameRepository(p.Repository, repository) && p.Commit == merge
|
||||
})
|
||||
if at < 0 {
|
||||
continue
|
||||
}
|
||||
p := m.Pulls[at]
|
||||
msg, err := bridge.MergedMessage(owner, repo, p.Number, m.Group, p.Head.Head, merge, trees, files)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the merged message of %s at %s cannot be built: %w", repository, merge, err)
|
||||
}
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, approved := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; approved && len(m.Approved) >= 2 {
|
||||
f.Group, f.Members = m.Group, groups.MembersOf(m)
|
||||
}
|
||||
out, err := forgemsg.EncodeForge(f)
|
||||
return out, err
|
||||
}
|
||||
return nil, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, so its pull request is not known", repository, merge)
|
||||
}
|
||||
|
||||
// read is the groups the gate holds with what the bridge kept of them: their ends in performed, and the last ask of
|
||||
// each in end-asks with the bridge's answer.
|
||||
func read(merges, performed, asks stdio.KeptState) ([]groups.Row, []string, error) {
|
||||
keys, err := merges.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the gate's merges cannot be listed: %w", err)
|
||||
}
|
||||
raws := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
e, err := merges.Get(k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
raws[k] = e.Value
|
||||
}
|
||||
pkeys, err := performed.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err)
|
||||
}
|
||||
published, ended := map[string]bool{}, map[string]uint64{}
|
||||
for _, k := range pkeys {
|
||||
switch {
|
||||
case strings.HasPrefix(k, "published."):
|
||||
published[k] = true
|
||||
case strings.HasPrefix(k, "groups."):
|
||||
if e, err := performed.Get(k); err == nil && e != nil {
|
||||
rev, _ := strconv.ParseUint(strings.TrimSpace(string(e.Value)), 10, 64)
|
||||
ended[strings.TrimPrefix(k, "groups.")] = rev
|
||||
}
|
||||
}
|
||||
}
|
||||
akeys, err := asks.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the bridge's state %s cannot be listed: %w", groups.EndAsksState, err)
|
||||
}
|
||||
askRaws := map[string][]byte{}
|
||||
for _, k := range akeys {
|
||||
if e, err := asks.Get(k); err == nil && e != nil {
|
||||
askRaws[k] = e.Value
|
||||
}
|
||||
}
|
||||
rows, unread := groups.Rows(raws, published, ended, askRaws)
|
||||
return rows, unread, nil
|
||||
}
|
||||
|
||||
// askEnd writes the end-asks value at key and waits, on a watch of the key, at most bound for the bridge's answer.
|
||||
func askEnd(s stdio.KeptState, bound time.Duration) asker {
|
||||
return func(key string, value []byte) (groups.EndAsk, error) {
|
||||
answered := make(chan groups.EndAsk, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
take := func(revision uint64, v []byte) {
|
||||
mu.Lock()
|
||||
mine := rev
|
||||
mu.Unlock()
|
||||
if mine == 0 || revision <= mine {
|
||||
return
|
||||
}
|
||||
if a, err := groups.DecodeEndAsk(v); err == nil && a.Done() {
|
||||
select {
|
||||
case answered <- a:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
if c.Op == "put" {
|
||||
take(c.Revision, c.Value)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return groups.EndAsk{}, fmt.Errorf("watching %s.%s: %w", groups.EndAsksState, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return groups.EndAsk{}, fmt.Errorf("the ask could not be written to %s: %w", groups.EndAsksBucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
if e, err := s.Get(key); err == nil && e != nil {
|
||||
take(e.Revision, e.Value)
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case a := <-answered:
|
||||
return a, nil
|
||||
case <-wait.Done():
|
||||
return groups.EndAsk{}, fmt.Errorf("the bridge did not answer within %s: its answer will be in %s.%s", bound, groups.EndAsksBucket, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// askCaller calls a tool as this bundle's module, through the runtime (stdio.Ask), the module's own invokes granting it.
|
||||
func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) {
|
||||
return stdio.Ask(key, body)
|
||||
}
|
||||
|
||||
func main() {
|
||||
merges, performed, asks := stdio.State(delivery.GateModule+".merges"), stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState)
|
||||
tools := Tools(askEnd(asks, 25*time.Second), time.Now,
|
||||
func() ([]groups.Row, []string, error) { return read(merges, performed, asks) },
|
||||
func(repository, merge string) (json.RawMessage, error) {
|
||||
keys, err := merges.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the gate's merges cannot be listed: %w", err)
|
||||
}
|
||||
raws := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
if e, err := merges.Get(k); err == nil && e != nil {
|
||||
raws[k] = e.Value
|
||||
}
|
||||
}
|
||||
ctx := context.Background()
|
||||
return MergedOf(raws, repository, merge, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller))
|
||||
})
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// 1585/T18, T19 (ADR 0633 §4): bridge_groups names each group the gate holds with its members' state and whether its end is
|
||||
// written (open or ended, with why), narrowed by group and by open; bridge_group_end_again writes an ask naming the group,
|
||||
// who asked and why, and answers what the bridge answered.
|
||||
func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) {
|
||||
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
|
||||
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
|
||||
enc := func(group string, state2 delivery.MergeState, c string) []byte {
|
||||
v, err := delivery.EncodeMerge(delivery.Merge{Group: group, Ask: "ask", Approved: map[string]string{"novox/a@1": h1, "novox/b@2": h2},
|
||||
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 1, Head: h1}, State: delivery.MergeMerged, Commit: c1},
|
||||
{Head: delivery.Head{Repository: "novox/b", Number: 2, Head: h2}, State: state2, Commit: c}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
lone, _ := delivery.EncodeMerge(delivery.Merge{Group: "solo", Ask: "ask", Approved: map[string]string{"novox/a@5": h1},
|
||||
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 5, Head: h1}, State: delivery.MergeMerged, Commit: c1}}})
|
||||
merges := map[string][]byte{"ended": enc("ended", delivery.MergeMerged, c2), "ready": enc("ready", delivery.MergeMerged, c2),
|
||||
"gathered": enc("gathered", delivery.MergeMerging, ""), "solo": lone, "bad": []byte("{")}
|
||||
performed := map[string]bool{}
|
||||
for _, c := range []string{c1, c2} {
|
||||
performed[groups.PublishedKey("novox/a", 1, c)], performed[groups.PublishedKey("novox/b", 2, c)] = true, true
|
||||
}
|
||||
// The state end-asks keeps the last ask of the group ready, answered, and one that does not read.
|
||||
answeredAsk, _ := json.Marshal(groups.EndAsk{V: 1, Group: "ready", Why: "lost", By: "lead@shanks", At: time.Date(2026, 11, 4, 22, 0, 0, 0, time.UTC),
|
||||
Written: true, Answer: "the end of the group is written again", Answered: time.Date(2026, 11, 4, 22, 0, 1, 0, time.UTC)})
|
||||
asks := map[string][]byte{"ready": answeredAsk, "gathered": []byte("{")}
|
||||
var asked groups.EndAsk
|
||||
var askedKey string
|
||||
ask := func(key string, value []byte) (groups.EndAsk, error) {
|
||||
askedKey = key
|
||||
a, err := groups.DecodeEndAsk(value)
|
||||
a.Written, a.Answer, a.Answered = true, "the end of the group is written again", time.Now()
|
||||
asked = a
|
||||
return a, err
|
||||
}
|
||||
tools := Tools(ask, func() time.Time { return time.Date(2026, 11, 4, 23, 0, 0, 0, time.UTC) },
|
||||
func() ([]groups.Row, []string, error) {
|
||||
rows, unread := groups.Rows(merges, performed, map[string]uint64{"ended": 7}, asks)
|
||||
return rows, unread, nil
|
||||
}, nil)
|
||||
call := func(name string, args map[string]any) any {
|
||||
for _, tl := range tools {
|
||||
if tl.Name == name {
|
||||
out, err := tl.Run(args)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", name)
|
||||
return nil
|
||||
}
|
||||
rowsOf := func(args map[string]any) map[string]groups.Row {
|
||||
raw, _ := json.Marshal(call(GroupsTool, args))
|
||||
var out struct {
|
||||
Groups []groups.Row `json:"groups"`
|
||||
Unread []string `json:"unread"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := map[string]groups.Row{}
|
||||
for _, r := range out.Groups {
|
||||
m[r.Group] = r
|
||||
}
|
||||
if len(args) == 0 && strings.Join(out.Unread, " ") != "bad end-asks.gathered" {
|
||||
t.Errorf("unread = %v, want bad and end-asks.gathered", out.Unread)
|
||||
}
|
||||
return m
|
||||
}
|
||||
all := rowsOf(map[string]any{})
|
||||
if len(all) != 3 || !all["ended"].Ended || all["ended"].EndedAt != 7 {
|
||||
t.Fatalf("groups = %+v", all)
|
||||
}
|
||||
if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published {
|
||||
t.Errorf("ready = %+v, want open, merged and published, its end the thing to say", r)
|
||||
}
|
||||
if a := all["ready"].LastEndAsk; a == nil || a.By != "lead@shanks" || a.Why != "lost" || !a.Written || a.Answer == "" || all["ended"].LastEndAsk != nil {
|
||||
t.Errorf("ready's last end ask = %+v, ended's %+v: want ready's ask and its answer, none for ended", a, all["ended"].LastEndAsk)
|
||||
}
|
||||
if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != string(delivery.MergeMerging) {
|
||||
t.Errorf("gathered = %+v", r)
|
||||
}
|
||||
if open := rowsOf(map[string]any{"open": true}); len(open) != 2 || open["ended"].Group != "" {
|
||||
t.Errorf("open = %+v, want ready and gathered", open)
|
||||
}
|
||||
if one := rowsOf(map[string]any{"group": "ready"}); len(one) != 1 {
|
||||
t.Errorf("group ready = %+v", one)
|
||||
}
|
||||
out := call(EndAgainTool, map[string]any{"group": "ready", "why": "the indexer never took it in"}).(map[string]any)
|
||||
if askedKey != "ready" || asked.Group != "ready" || asked.Why != "the indexer never took it in" || out["written"] != true {
|
||||
t.Errorf("the ask %q %+v, answered %v", askedKey, asked, out)
|
||||
}
|
||||
for _, tl := range tools {
|
||||
if tl.Name == EndAgainTool {
|
||||
if _, err := tl.Run(map[string]any{}); err == nil {
|
||||
t.Error("an ask naming no group was taken")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1585: bridge_merged_message answers the merged message as the bridge publishes it: for a lone pull request the same bytes
|
||||
// the event gives through Convert, for a member of a delivery group those with its group and members set as the process
|
||||
// sets them; a merge the gate does not hold, or a forge that does not answer, is said.
|
||||
t.Run("bridge_merged_message answers the merged message as published", func(t *testing.T) {
|
||||
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
|
||||
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
|
||||
trees := func(owner, repo, sha string) (string, map[string]string, error) {
|
||||
return "root-" + sha[:4], map[string]string{"": "r", "modules/a": "ta-" + sha[:4]}, nil
|
||||
}
|
||||
files := func(owner, repo, path, ref string) ([]byte, error) {
|
||||
if strings.HasSuffix(path, "module.json") {
|
||||
return []byte(`{"module":"a"}`), nil
|
||||
}
|
||||
return nil, bridge.ErrNotFound
|
||||
}
|
||||
published := func(repo string, number int, head, branch, merge string) forgemsg.Forge {
|
||||
body, _ := json.Marshal(map[string]any{"owner": "novox", "repo": repo, "number": number, "head": branch, "head_sha": head, "base": "main", "merge_commit_sha": merge})
|
||||
msg, err := bridge.Convert(bridge.MergedSubject, body, trees, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
lone, _ := delivery.EncodeMerge(delivery.Merge{Group: "solo", Ask: "a", Approved: map[string]string{"novox/a@5": h1},
|
||||
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 5, Head: h1}, State: delivery.MergeMerged, Commit: c1}}})
|
||||
group, _ := delivery.EncodeMerge(delivery.Merge{Group: "feat/g", Ask: "a", Approved: map[string]string{"novox/a@6": h1, "novox/b@7": h2},
|
||||
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 6, Head: h1}, State: delivery.MergeMerged, Commit: c1},
|
||||
{Head: delivery.Head{Repository: "novox/b", Number: 7, Head: h2}, State: delivery.MergeMerging}}})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
raws map[string][]byte
|
||||
repository string
|
||||
merge string
|
||||
want forgemsg.Forge
|
||||
fail string
|
||||
}{
|
||||
{name: "a lone pull request", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c1, want: published("a", 5, h1, "solo", c1)},
|
||||
{name: "a member of a group", raws: map[string][]byte{"feat~g": group}, repository: "novox/a", merge: c1, want: func() forgemsg.Forge {
|
||||
f := published("a", 6, h1, "feat/g", c1)
|
||||
f.Group, f.Members = "feat/g", []delivery.Head{{Repository: "novox/a", Number: 6, Head: h1}, {Repository: "novox/b", Number: 7, Head: h2}}
|
||||
return f
|
||||
}()},
|
||||
{name: "a merge the gate does not hold", raws: map[string][]byte{"solo": lone, "bad": []byte("{")}, repository: "novox/a", merge: c2, fail: "hold no merge"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
raw, err := MergedOf(c.raws, c.repository, c.merge, trees, files)
|
||||
if c.fail != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.fail) {
|
||||
t.Fatalf("err %v, want %q", err, c.fail)
|
||||
}
|
||||
return
|
||||
}
|
||||
want, _ := forgemsg.EncodeForge(c.want)
|
||||
if err != nil || !bytes.Equal(raw, want) {
|
||||
t.Fatalf("err %v, answer\n%s\nwant the published message\n%s", err, raw, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := MergedOf(map[string][]byte{"solo": lone}, "novox/a", c1, func(string, string, string) (string, map[string]string, error) {
|
||||
return "", nil, errors.New("the forge does not answer")
|
||||
}, files); err == nil || !strings.Contains(err.Error(), "does not answer") {
|
||||
t.Errorf("a forge that does not answer: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -66,58 +65,12 @@ func callerOn(nc *nats.Conn, user string) caller {
|
||||
}
|
||||
}
|
||||
|
||||
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner,
|
||||
// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4).
|
||||
const commitTreesTool = "gitea.gitea_commit_trees"
|
||||
|
||||
// treesVia asks the forge for a commit's trees through call.
|
||||
func treesVia(ctx context.Context, call caller) bridge.Trees {
|
||||
return func(owner, repo, sha string) (string, map[string]string, error) {
|
||||
raw, err := call(ctx, commitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var a struct {
|
||||
Root string `json:"root"`
|
||||
Trees map[string]string `json:"trees"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
|
||||
}
|
||||
if a.Root == "" {
|
||||
return "", nil, fmt.Errorf("the answer of %s names no root tree", commitTreesTool)
|
||||
}
|
||||
return a.Root, a.Trees, nil
|
||||
}
|
||||
return bridge.TreesVia(ctx, bridge.Caller(call))
|
||||
}
|
||||
|
||||
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
|
||||
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
|
||||
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
|
||||
const getFileTool = "gitea.gitea_get_file"
|
||||
|
||||
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
|
||||
// recognises it as /: 404 /).
|
||||
var notFound = regexp.MustCompile(`: 404 `)
|
||||
|
||||
// filesVia asks the forge for a file through call. A file the forge says is not there is bridge.ErrNotFound.
|
||||
func filesVia(ctx context.Context, call caller) bridge.Files {
|
||||
return func(owner, repo, path, ref string) ([]byte, error) {
|
||||
raw, err := call(ctx, getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
|
||||
if err != nil {
|
||||
if notFound.MatchString(err.Error()) {
|
||||
return nil, fmt.Errorf("%w: %v", bridge.ErrNotFound, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
|
||||
}
|
||||
return []byte(*a.File.Content), nil
|
||||
}
|
||||
return bridge.FilesVia(ctx, bridge.Caller(call))
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -120,8 +121,8 @@ func TestARefusedActIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1341/AC
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.refused" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
m, err := delivery.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != delivery.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" {
|
||||
m, err := forgemsg.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != forgemsg.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" {
|
||||
t.Errorf("message %+v err %v", m, err)
|
||||
}
|
||||
f.refuse = nil
|
||||
@@ -188,8 +189,8 @@ func TestARefusedMergeIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1342/
|
||||
if len(f.calls) != 1 || len(pub.data) != 1 {
|
||||
t.Fatalf("calls %d messages %d, want one each", len(f.calls), len(pub.data))
|
||||
}
|
||||
m, err := delivery.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != delivery.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA ||
|
||||
m, err := forgemsg.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != forgemsg.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA ||
|
||||
m.Act != "merge" || m.Words != "novox/mesh-delivery#12 is not merged: its head moved" {
|
||||
t.Errorf("message %+v err %v", m, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// endAsked answers one value of the state end-asks (the tool bridge_group_end_again, ADR 0633 §4): a value not yet answered
|
||||
// says the group's end again from the gate's merges, with the ask's time appended to the message id so the stream's duplicate
|
||||
// window does not drop it, and is written back by compare-and-set at its revision with whether it was written. A value
|
||||
// answered already (this bridge's own write) is left.
|
||||
func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte) {
|
||||
a, err := groups.DecodeEndAsk(value)
|
||||
if err != nil {
|
||||
g.logger.Printf("%s.%s holds what is not an end ask, left: %v", groups.EndAsksState, key, err)
|
||||
return
|
||||
}
|
||||
if a.Done() {
|
||||
return
|
||||
}
|
||||
a = g.answerEnd(ctx, key, a, time.Now())
|
||||
g.logger.Printf("the end of the group %q said again by %s (%s): %s", a.Group, a.By, a.Why, a.Answer)
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
g.logger.Printf("%s.%s: the answer could not be written: %v", groups.EndAsksState, key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// readMerges reads one key of the gate's merges state.
|
||||
func (g *grouper) readMerges(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
if g.merges == nil {
|
||||
return nil, 0, false, errors.New("the bridge does not read the gate's merges (its bucket was not there at the start)")
|
||||
}
|
||||
value, rev, found, err := g.merges(ctx, key)
|
||||
return value, rev, found, err
|
||||
}
|
||||
|
||||
// answerEnd says the group's end again for an ask and returns the ask with its answer.
|
||||
func (g *grouper) answerEnd(ctx context.Context, key string, a groups.EndAsk, at time.Time) groups.EndAsk {
|
||||
a.Answered = at.UTC()
|
||||
switch merges, mrev, found, err := g.readMerges(ctx, key); {
|
||||
case err != nil:
|
||||
a.Answer = "the gate's merges " + key + " cannot be read: " + err.Error()
|
||||
case !found:
|
||||
a.Answer = "the gate holds no merge of the group " + a.Group
|
||||
case g.end(ctx, key, mrev, merges, groups.AgainStamp(a.At)):
|
||||
a.Written, a.Answer = true, "the end of the group is written again"
|
||||
default:
|
||||
a.Answer = "the end is not written: a member is not merged or its merged message is not written, or FORGE refused it (the bridge's journal says which)"
|
||||
}
|
||||
return a
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"slices"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
groupkeys "mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// A delivery group's merges reach the indexer together (novox/hq ADR 0633 §4): the bridge sets the group on each
|
||||
// member's merged message, from the gate's merges state it reads (ADR 0437 §8), and once every approved member is
|
||||
// merged and its merged message written on FORGE it says the group's end, forge.group.merged, once.
|
||||
//
|
||||
// What it keeps of it is in its state performed, beside the gate's keys it acted on: `published.<hash>` for each
|
||||
// member's merged message written (the hash is the first 16 hex digits of the sha256 of `<repository>#<number>@<merge>`),
|
||||
// and `groups.<merges key>` once the group's end is written, its value the revision of the gate's merges key it was
|
||||
// worked out from. A group whose end is not written yet is open (bridge_groups reads them).
|
||||
|
||||
// merges reads one key of the gate's merges state.
|
||||
type mergesRead func(ctx context.Context, key string) (value []byte, rev uint64, found bool, err error)
|
||||
|
||||
// grouper sets a member's group and says a group's end.
|
||||
type grouper struct {
|
||||
merges mergesRead
|
||||
send func(ctx context.Context, subject string, data []byte, id string) error // writes on FORGE with the message id
|
||||
done revisions
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// sendVia writes on FORGE through pub, with the message id.
|
||||
func sendVia(pub publisher) func(context.Context, string, []byte, string) error {
|
||||
return func(ctx context.Context, subject string, data []byte, id string) error {
|
||||
_, err := pub.Publish(ctx, subject, data, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func kvMerges(kv jetstream.KeyValue) mergesRead {
|
||||
return func(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// groupOf is the gate's merge of the group a merged message's branch names, when the message's pull request is one of
|
||||
// its approved members and the group has more than one: a lone pull request is no group.
|
||||
func (g *grouper) groupOf(ctx context.Context, f forgemsg.Forge) (delivery.Merge, uint64, bool) {
|
||||
if g == nil || g.merges == nil || f.Branch == "" {
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
key, err := delivery.MergesKey(f.Branch)
|
||||
if err != nil {
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
value, rev, found, err := g.merges(ctx, key)
|
||||
if err != nil {
|
||||
g.logger.Printf("%s#%d: the gate's merges.%s cannot be read, so its merged message names no group: %v", f.Repository, f.Number, key, err)
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
if !found {
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
m, err := delivery.DecodeMerge(value)
|
||||
if err != nil {
|
||||
g.logger.Printf("%s#%d: the gate's merges.%s is not a merge, so its merged message names no group: %v", f.Repository, f.Number, key, err)
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
if _, ok := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; !ok || len(m.Approved) < 2 {
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
return m, rev, true
|
||||
}
|
||||
|
||||
// member sets the group and its approved members on a merged message's body (ADR 0633 §4); any other body, and a
|
||||
// merge of no group, is left as it is.
|
||||
func (g *grouper) member(ctx context.Context, data []byte) ([]byte, forgemsg.Forge, bool) {
|
||||
f, err := forgemsg.DecodeForge(data)
|
||||
if err != nil || f.Kind != forgemsg.ForgeMerged {
|
||||
return data, f, false
|
||||
}
|
||||
m, _, ok := g.groupOf(ctx, f)
|
||||
if !ok {
|
||||
return data, f, false
|
||||
}
|
||||
f.Group, f.Members = m.Group, groupkeys.MembersOf(m)
|
||||
out, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.Group, err)
|
||||
return data, f, false
|
||||
}
|
||||
return out, f, true
|
||||
}
|
||||
|
||||
// published keeps that a member's merged message is written, and says the group's end when it was the last.
|
||||
func (g *grouper) published(ctx context.Context, f forgemsg.Forge) {
|
||||
if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number, f.Merge), 1); err != nil {
|
||||
g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err)
|
||||
return
|
||||
}
|
||||
key, err := delivery.MergesKey(f.Branch)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
value, rev, found, err := g.merges(ctx, key)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
g.ended(ctx, key, rev, value)
|
||||
}
|
||||
|
||||
// ended says the group's end (forge.group.merged) once every approved member of the gate's merge is merged and its
|
||||
// merged message written, after the last of them; never on a refusal (ADR 0441 §3: a held group merges on from it).
|
||||
// It is written once per group: `groups.<key>` in performed says it was. again says it is written again by hand
|
||||
// (bridge_group_end_again), with the time appended to the message id, so the duplicate window does not drop it.
|
||||
func (g *grouper) ended(ctx context.Context, key string, rev uint64, value []byte) {
|
||||
g.end(ctx, key, rev, value, "")
|
||||
}
|
||||
|
||||
func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, again string) bool {
|
||||
m, err := delivery.DecodeMerge(value)
|
||||
if err != nil || len(m.Approved) < 2 {
|
||||
return false
|
||||
}
|
||||
if again == "" {
|
||||
if _, seen, err := g.done.Get(ctx, groupkeys.Key(key)); err == nil && seen {
|
||||
return false
|
||||
}
|
||||
}
|
||||
end := forgemsg.ForgeGroupEnd{Group: m.Group}
|
||||
for _, member := range groupkeys.MembersOf(m) {
|
||||
at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool {
|
||||
return delivery.SameRepository(p.Repository, member.Repository) && p.Number == member.Number
|
||||
})
|
||||
if at < 0 || m.Pulls[at].Commit == "" {
|
||||
return false // not every member is merged (a merge commit is written only for a merged pull): the group stays gathered
|
||||
}
|
||||
if _, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok {
|
||||
return false // its merged message is not written yet: said when it is (published)
|
||||
}
|
||||
end.Members = append(end.Members, forgemsg.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit})
|
||||
}
|
||||
// The members in the order the gate merged them, so the last commit names the end.
|
||||
slices.SortStableFunc(end.Members, func(a, b forgemsg.GroupMerge) int {
|
||||
return slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == a.Merge }) -
|
||||
slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == b.Merge })
|
||||
})
|
||||
data, err := forgemsg.EncodeForgeGroupEnd(end)
|
||||
if err != nil {
|
||||
g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err)
|
||||
return false
|
||||
}
|
||||
id := groupkeys.EndID(m.Group, end.Members[len(end.Members)-1].Merge, "")
|
||||
if again != "" {
|
||||
id += "." + again
|
||||
}
|
||||
if err := g.send(ctx, forgeSubject(forgemsg.ForgeGroupEndEvent), data, id); err != nil {
|
||||
g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err)
|
||||
return false
|
||||
}
|
||||
if err := g.done.Put(ctx, groupkeys.Key(key), rev); err != nil {
|
||||
g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.Group, err)
|
||||
}
|
||||
g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.Group, len(end.Members), end.Members[len(end.Members)-1].Merge)
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
groupkeys "mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// idPub records the subjects, bodies and message ids it is handed.
|
||||
type idPub struct {
|
||||
subjects, ids []string
|
||||
data [][]byte
|
||||
}
|
||||
|
||||
func (p *idPub) send(_ context.Context, subject string, data []byte, id string) error {
|
||||
p.subjects, p.data, p.ids = append(p.subjects, subject), append(p.data, data), append(p.ids, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17, T19 (ADR 0633 §4)
|
||||
ctx := context.Background()
|
||||
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
|
||||
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
|
||||
merge := func(states ...delivery.MergePull) []byte {
|
||||
v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Pulls: states,
|
||||
Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
sdk := func(state delivery.MergeState, commit, words string) delivery.MergePull {
|
||||
return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-sdk", Number: 37, Head: h1}, State: state, Commit: commit, Words: words}
|
||||
}
|
||||
dlv := func(state delivery.MergeState, commit string) delivery.MergePull {
|
||||
return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: state, Commit: commit}
|
||||
}
|
||||
gate := map[string][]byte{}
|
||||
var logs bytes.Buffer
|
||||
pub, done := &idPub{}, memRevisions{}
|
||||
g := &grouper{send: pub.send, done: done, logger: log.New(&logs, "", 0),
|
||||
merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := gate[key]
|
||||
return v, 7, ok, nil
|
||||
}}
|
||||
mergedOf := func(repo string, number int, head, commit string) []byte {
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
// A lone pull request's merged message carries no group.
|
||||
if _, f, ok := g.member(ctx, mergedOf("novox/hq", 4, h1, c1)); ok || f.Group != "" {
|
||||
t.Errorf("a lone pull request names the group %q", f.Group)
|
||||
}
|
||||
// The first member: its message names the group and both members; the group is not ended.
|
||||
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerging, ""))
|
||||
data, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1))
|
||||
if back, err := forgemsg.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 {
|
||||
t.Fatalf("a member's message: %+v %v", back, err)
|
||||
}
|
||||
g.published(ctx, f)
|
||||
// A refusal of the other member is no end (ADR 0441 §3: the group merges on from it later).
|
||||
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: delivery.MergeRefused, Words: "405"})
|
||||
g.ended(ctx, "feat/take-in", 8, gate["feat/take-in"])
|
||||
// Merged on at the gate before its merged message is written: still no end; then its message ends the group.
|
||||
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerged, c2))
|
||||
g.ended(ctx, "feat/take-in", 9, gate["feat/take-in"])
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Fatalf("an end before every member's merged message is written: %v", pub.subjects)
|
||||
}
|
||||
_, f, _ = g.member(ctx, mergedOf("novox/mesh-delivery", 9, h2, c2))
|
||||
g.published(ctx, f)
|
||||
g.ended(ctx, "feat/take-in", 10, gate["feat/take-in"]) // a later write of the gate says no second end
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.group.merged" || pub.ids[0] != "group-end.feat/take-in."+c2 {
|
||||
t.Fatalf("ends %v ids %v, want one forge.group.merged with id group-end.feat/take-in.%s", pub.subjects, pub.ids, c2)
|
||||
}
|
||||
end, err := forgemsg.DecodeForgeGroupEnd(pub.data[0])
|
||||
if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 {
|
||||
t.Errorf("the end = %+v %v, want both members in their merge order", end, err)
|
||||
}
|
||||
if rev, ok := done[groupkeys.Key("feat/take-in")]; !ok || rev != 7 {
|
||||
t.Errorf("performed %v: want groups.feat/take-in kept", done)
|
||||
}
|
||||
// Again by hand: written once more, its id not the first's.
|
||||
if !g.end(ctx, "feat/take-in", 10, gate["feat/take-in"], "20261104T2300") || pub.ids[1] == pub.ids[0] {
|
||||
t.Errorf("the end again by hand: ids %v", pub.ids)
|
||||
}
|
||||
// 1585/T19: an ask to say a group's end again is answered by the bridge from the gate's merges, with the ask's time on
|
||||
// the message id; not written while a member is not merged, with the reason; not for a group the gate does not hold.
|
||||
pull := func(repo string, n int, head string, state delivery.MergeState, commit string) delivery.MergePull {
|
||||
return delivery.MergePull{Head: delivery.Head{Repository: repo, Number: n, Head: head}, State: state, Commit: commit}
|
||||
}
|
||||
askMerge := func(second delivery.MergeState, commit2 string) []byte {
|
||||
v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2},
|
||||
Pulls: []delivery.MergePull{pull("novox/mesh-sdk", 37, h1, delivery.MergeMerged, c1), pull("novox/mesh-delivery", 9, h2, second, commit2)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
ask := groupkeys.EndAsk{V: 1, Group: "feat/take-in", By: "operator@shanks", Why: "the indexer never took it in", At: time.Date(2026, 11, 4, 23, 0, 0, 123, time.UTC)}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
gate map[string][]byte
|
||||
published bool
|
||||
written bool
|
||||
answer string
|
||||
}{
|
||||
{"merged and published: written again", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, true, true, "written again"},
|
||||
{"a member's message not written", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, false, false, "not written"},
|
||||
{"a member not merged", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerging, "")}, true, false, "not written"},
|
||||
{"no merge at the gate", map[string][]byte{}, true, false, "holds no merge"},
|
||||
} {
|
||||
pub, done := &idPub{}, memRevisions{}
|
||||
if tc.published {
|
||||
done[groupkeys.PublishedKey("novox/mesh-sdk", 37, c1)], done[groupkeys.PublishedKey("novox/mesh-delivery", 9, c2)] = 1, 1
|
||||
}
|
||||
g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0),
|
||||
merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := tc.gate[key]
|
||||
return v, 4, ok, nil
|
||||
}}
|
||||
got := g.answerEnd(ctx, "feat/take-in", ask, ask.At.Add(time.Second))
|
||||
if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] {
|
||||
t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids)
|
||||
}
|
||||
if tc.written && pub.ids[0] != groupkeys.EndID("feat/take-in", c2, groupkeys.AgainStamp(ask.At)) {
|
||||
t.Errorf("%s: the id %s does not carry the ask's time", tc.name, pub.ids[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
-13
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -36,7 +37,7 @@ import (
|
||||
//
|
||||
// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal
|
||||
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls,
|
||||
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
|
||||
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, performed, inputs and end-asks (ADR 0633 §5's hand end), and the stream forge; `**` is written `>` on the wire (modstreams.go
|
||||
// WirePattern). The consumer's durable is <node>_<module> on EVENTS (nats.go consumerDurable). The controller's own
|
||||
// user keeps all rights, as it has.
|
||||
const busConf = `
|
||||
@@ -83,6 +84,12 @@ accounts {
|
||||
"$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>"
|
||||
"$KV.mesh-delivery-forge-bridge_performed.>"
|
||||
"$KV.mesh-delivery-forge-bridge_inputs.>"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_end-asks"
|
||||
"$JS.FC.KV_mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$KV.mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>"
|
||||
@@ -321,13 +328,13 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(raw.Data)
|
||||
f, err := forgemsg.DecodeForge(raw.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
|
||||
@@ -416,16 +423,16 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(raw.Data)
|
||||
f, err := forgemsg.DecodeForge(raw.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
|
||||
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
|
||||
// No mesh package: only the indexer computes what a fingerprint takes (ADR 0593 §2).
|
||||
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
select {
|
||||
@@ -478,7 +485,7 @@ func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err := delivery.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
|
||||
if f, err := forgemsg.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
|
||||
t.Errorf("%s: %+v err %v", repo, f, err)
|
||||
}
|
||||
}
|
||||
@@ -548,20 +555,20 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes
|
||||
t.Fatalf("FORGE holds %d messages (%v), want %d", info.State.Msgs, err, want)
|
||||
}
|
||||
}
|
||||
named := func(seq uint64) []delivery.OpenPull {
|
||||
named := func(seq uint64) []forgemsg.OpenPull {
|
||||
t.Helper()
|
||||
raw, err := forge.GetMsg(b.ctx, seq)
|
||||
if err != nil || raw.Subject != openSubject {
|
||||
t.Fatalf("message %d: %v %v, want the open message", seq, raw, err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(raw.Data)
|
||||
f, err := forgemsg.DecodeForge(raw.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f.Open
|
||||
}
|
||||
bothOpen := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
same := func(got []delivery.OpenPull) bool {
|
||||
bothOpen := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
same := func(got []forgemsg.OpenPull) bool {
|
||||
return len(got) == 2 && got[0] == bothOpen[0] && got[1] == bothOpen[1]
|
||||
}
|
||||
|
||||
@@ -713,7 +720,7 @@ func forgeMerged(t *testing.T, b *bed) []string {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if f, err := delivery.DecodeForge(raw.Data); err == nil && f.Kind == delivery.ForgeMerged {
|
||||
if f, err := forgemsg.DecodeForge(raw.Data); err == nil && f.Kind == forgemsg.ForgeMerged {
|
||||
out = append(out, f.Repository)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,9 +21,11 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
groupkeys "mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// eventsStream is the shared stream every module's events are on; the module's consumer on it is the durable
|
||||
@@ -68,16 +70,18 @@ type publisher interface {
|
||||
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, logger *log.Logger) {
|
||||
handleGrouped(ctx, m, pub, trees, files, kept, nil, logger)
|
||||
}
|
||||
|
||||
// handleGrouped is handle; a merged message of a delivery group's member names its group (g, ADR 0633 §4; nil names
|
||||
// none), and the member's merged message written may end the group.
|
||||
func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, g *grouper, logger *log.Logger) {
|
||||
msg, err := bridge.ConvertKept(m.Subject(), m.Data(), trees, files, kept)
|
||||
switch {
|
||||
case errors.Is(err, bridge.ErrRetry):
|
||||
logger.Printf("%v; heard again in %s", err, retryDelay)
|
||||
_ = m.NakWithDelay(retryDelay)
|
||||
return
|
||||
case errors.Is(err, bridge.ErrNoModules):
|
||||
logger.Printf("%v; no merged message", err)
|
||||
_ = m.Ack()
|
||||
return
|
||||
case errors.Is(err, bridge.ErrNotHandled):
|
||||
logger.Printf("%v; nothing written", err)
|
||||
_ = m.Ack()
|
||||
@@ -92,10 +96,18 @@ func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees,
|
||||
if id := m.Headers().Get("x-event-id"); id != "" {
|
||||
opts = append(opts, jetstream.WithMsgID(id))
|
||||
}
|
||||
var member forgemsg.Forge
|
||||
grouped := false
|
||||
if g != nil {
|
||||
msg.Data, member, grouped = g.member(ctx, msg.Data)
|
||||
}
|
||||
if _, err := pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, opts...); err != nil {
|
||||
logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err)
|
||||
return
|
||||
}
|
||||
if grouped {
|
||||
g.published(ctx, member)
|
||||
}
|
||||
if msg.Said != "" {
|
||||
logger.Printf("%s: %s", m.Subject(), msg.Said)
|
||||
}
|
||||
@@ -140,17 +152,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
|
||||
// A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the gate's merges.
|
||||
groups := &grouper{send: sendVia(js), done: kvRevisions{performedKV}, logger: logger}
|
||||
watching := make(chan error, 3)
|
||||
for _, w := range []struct {
|
||||
bucket string
|
||||
react func(ctx context.Context, key string, rev uint64, value []byte)
|
||||
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
|
||||
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, func(ctx context.Context, key string, rev uint64, value []byte) {
|
||||
perf.merges(ctx, key, rev, value)
|
||||
groups.ended(ctx, key, rev, value)
|
||||
}}} {
|
||||
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started
|
||||
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945).
|
||||
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf)
|
||||
if kv == nil {
|
||||
continue
|
||||
}
|
||||
if w.bucket == delivery.MergesBucket {
|
||||
groups.merges = kvMerges(kv)
|
||||
}
|
||||
go func() {
|
||||
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
@@ -159,6 +179,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
})
|
||||
}()
|
||||
}
|
||||
// A person says a group's end again with bridge_group_end_again, which writes the state end-asks (ADR 0633 §4): watched, and
|
||||
// answered there; a state not there at this start is answered from the next start.
|
||||
if kv := process.OptionalBucket(ctx, js, groupkeys.EndAsksBucket, "bridge_group_end_again is not answered until the bridge starts again", logger.Printf); kv != nil {
|
||||
go func() {
|
||||
err := watch(ctx, kv, func(key string, rev uint64, value []byte) { groups.endAsked(ctx, kv, key, rev, value) })
|
||||
if err != nil && ctx.Err() == nil {
|
||||
logger.Printf("the end asks are no longer watched, so bridge_group_end_again is not answered until the bridge starts again: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err := said(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -175,7 +205,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Bounded: the turn is held, and no sync or other event goes on without it.
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, logger)
|
||||
handleGrouped(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, groups, logger)
|
||||
})
|
||||
}, logger.Printf)
|
||||
if err != nil {
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
@@ -85,7 +85,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
f, err := delivery.DecodeForge(pub.data[0])
|
||||
f, err := forgemsg.DecodeForge(pub.data[0])
|
||||
if err != nil || f.Number != 12 || f.Branch != "feat/x" || f.Head != head || f.Action != "updated" || !f.DirsKnown {
|
||||
t.Errorf("message %+v err %v", f, err)
|
||||
}
|
||||
@@ -107,11 +107,10 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
if !strings.Contains(logs.String(), "mesh.mod.gitea.event.pull.updated") || !strings.Contains(logs.String(), "head") {
|
||||
t.Errorf("log %q names neither the subject nor why", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
|
||||
// 1235: an unwritten message is not acked (a failed write is redelivered by the server, not retried here); merged here
|
||||
// from a test of its own, for the test budget.
|
||||
logs.Reset()
|
||||
m, pub = &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
|
||||
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
|
||||
if m.acked != 0 || m.termd != 0 {
|
||||
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
|
||||
@@ -173,7 +172,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
f, err := delivery.DecodeForge(pub.data[0])
|
||||
f, err := forgemsg.DecodeForge(pub.data[0])
|
||||
if err != nil || f.Root != "r0" || f.Trees[""] != "t0" || f.Trees["modules/a"] != "t1" || f.Merge == "" {
|
||||
t.Errorf("message %+v err %v", f, err)
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
@@ -41,7 +41,7 @@ func (r *recorder) Publish(ctx context.Context, subject string, payload []byte,
|
||||
if err != nil {
|
||||
return ack, err
|
||||
}
|
||||
f, derr := delivery.DecodeForge(payload)
|
||||
f, derr := forgemsg.DecodeForge(payload)
|
||||
if derr != nil {
|
||||
return ack, nil
|
||||
}
|
||||
@@ -101,7 +101,7 @@ func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore,
|
||||
}
|
||||
var differing []bridge.Message
|
||||
for _, m := range msgs {
|
||||
f, err := delivery.DecodeForge(m.Data)
|
||||
f, err := forgemsg.DecodeForge(m.Data)
|
||||
if err != nil {
|
||||
differing = append(differing, m)
|
||||
continue
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds its entries inside {pulls}.
|
||||
@@ -43,7 +43,7 @@ func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1,
|
||||
}
|
||||
want := []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.pull"}
|
||||
for i, w := range want {
|
||||
f, err := delivery.DecodeForge(pub.data[i])
|
||||
f, err := forgemsg.DecodeForge(pub.data[i])
|
||||
if pub.subjects[i] != w || err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
|
||||
t.Errorf("message %d on %s = %+v err %v", i, pub.subjects[i], f, err)
|
||||
}
|
||||
@@ -112,8 +112,8 @@ func TestSyncOpenWritesPullsForNewHeadsThenOneOpenMessageNamingAll(t *testing.T)
|
||||
if pub.subjects[1] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.open" {
|
||||
t.Fatalf("last message on %s, want the open message after the pull messages", pub.subjects[1])
|
||||
}
|
||||
f, err := delivery.DecodeForge(pub.data[1])
|
||||
want := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
f, err := forgemsg.DecodeForge(pub.data[1])
|
||||
want := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
if err != nil || len(f.Open) != 2 || f.Open[0] != want[0] || f.Open[1] != want[1] {
|
||||
t.Errorf("open message = %+v err %v, want it to name %v, the pull request whose head did not differ among them", f, err, want)
|
||||
}
|
||||
@@ -140,10 +140,10 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6
|
||||
pub := &fakePub{}
|
||||
heads := newHeads()
|
||||
r := &recorder{pub: pub, heads: heads, logger: log.New(&bytes.Buffer{}, "", 0)}
|
||||
write := func(f delivery.Forge) {
|
||||
write := func(f forgemsg.Forge) {
|
||||
t.Helper()
|
||||
ev, _ := delivery.ForgeEvent(f.Repository, f.Kind)
|
||||
raw, err := delivery.EncodeForge(f)
|
||||
ev, _ := forgemsg.ForgeEvent(f.Repository, f.Kind)
|
||||
raw, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -151,19 +151,19 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p := delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated}
|
||||
p := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated}
|
||||
write(p)
|
||||
if heads.get("novox.mesh-delivery.12") != head {
|
||||
t.Fatalf("bucket %v", heads.vals)
|
||||
}
|
||||
p.Action = delivery.PullClosed
|
||||
p.Action = forgemsg.PullClosed
|
||||
write(p)
|
||||
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
|
||||
t.Errorf("a closed pull request is still kept: %v", heads.vals)
|
||||
}
|
||||
p.Action = delivery.PullUpdated
|
||||
p.Action = forgemsg.PullUpdated
|
||||
write(p)
|
||||
write(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
|
||||
write(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
|
||||
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
|
||||
t.Errorf("a merged pull request is still kept: %v", heads.vals)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
// Package forgemsg is the forge bridge's contract on its own stream `forge` (novox/hq ADR 0633 §4): its messages (Forge,
|
||||
// one per pull request, merge, refusal or list of open pull requests) and a delivery group's end (ForgeGroupEnd). The
|
||||
// forge bridge writes them; the indexer and the gate read them. It is the forge bridge's own package: one module owns
|
||||
// these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04).
|
||||
package forgemsg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Version is the version of every value this package writes and the only one it reads: mesh-sdk's delivery values'.
|
||||
const Version = delivery.Version
|
||||
|
||||
// ForgeStream is the forge bridge's stream as its manifest declares it; the broker derives the real names.
|
||||
const ForgeStream = "forge"
|
||||
|
||||
// The kinds of forge message, the last part of a forge event's name.
|
||||
const (
|
||||
ForgePull = "pull"
|
||||
ForgeMerged = "merged"
|
||||
ForgeRefused = "refused"
|
||||
ForgeOpen = "open"
|
||||
)
|
||||
|
||||
// OpenPull names an open pull request into the trunk: the repository as `<owner>/<name>` and the number.
|
||||
type OpenPull struct {
|
||||
Repository string `json:"repository"`
|
||||
Number int `json:"number"`
|
||||
}
|
||||
|
||||
// OpenEvent is the name of an open message: `forge.open`. It names no repository: it says every open pull request of
|
||||
// every repository the forge bridge reads (novox/hq ADR 0443 §5).
|
||||
func OpenEvent() string { return "forge." + ForgeOpen }
|
||||
|
||||
// The actions of a pull message.
|
||||
const (
|
||||
PullOpened = "opened"
|
||||
PullUpdated = "updated"
|
||||
PullClosed = "closed"
|
||||
)
|
||||
|
||||
// checkRepository refuses a repository that is not `<owner>/<name>`: exactly one slash, each part a valid name.
|
||||
func checkRepository(repository string) error {
|
||||
owner, name, ok := strings.Cut(repository, "/")
|
||||
if !ok || strings.Contains(name, "/") {
|
||||
return fmt.Errorf("delivery: the repository %q is not <owner>/<name>, with exactly one slash", repository)
|
||||
}
|
||||
if err := checkName("repository owner", owner); err != nil {
|
||||
return fmt.Errorf("delivery: the repository %q: %w", repository, err)
|
||||
}
|
||||
if err := checkName("repository name", name); err != nil {
|
||||
return fmt.Errorf("delivery: the repository %q: %w", repository, err)
|
||||
}
|
||||
if strings.Contains(owner, "/") {
|
||||
return fmt.Errorf("delivery: the repository %q is not <owner>/<name>, with exactly one slash", repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkForgeKind(kind string) error {
|
||||
switch kind {
|
||||
case ForgePull, ForgeMerged, ForgeRefused, ForgeOpen:
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("delivery: the forge kind %q is not pull, merged, refused or open", kind)
|
||||
}
|
||||
|
||||
// ForgeEvent is the name of a forge message of the given kind: `forge.<owner>.<name>.<kind>`.
|
||||
func ForgeEvent(repository, kind string) (string, error) {
|
||||
if err := checkForgeKind(kind); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if kind == ForgeOpen {
|
||||
return "", fmt.Errorf("delivery: an open message names no repository, its event is %s", OpenEvent())
|
||||
}
|
||||
if err := checkRepository(repository); err != nil {
|
||||
return "", err
|
||||
}
|
||||
owner, name, _ := strings.Cut(repository, "/")
|
||||
return "forge." + owner + "." + name + "." + kind, nil
|
||||
}
|
||||
|
||||
// ParseForgeEvent is the inverse of ForgeEvent.
|
||||
func ParseForgeEvent(event string) (repository, kind string, err error) {
|
||||
if event == OpenEvent() {
|
||||
return "", ForgeOpen, nil
|
||||
}
|
||||
parts := strings.Split(event, ".")
|
||||
if len(parts) != 4 || parts[0] != "forge" {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a forge event, forge.<owner>.<name>.<kind>", event)
|
||||
}
|
||||
repository = parts[1] + "/" + parts[2]
|
||||
if _, err := ForgeEvent(repository, parts[3]); err != nil {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a forge event: %w", event, err)
|
||||
}
|
||||
return repository, parts[3], nil
|
||||
}
|
||||
|
||||
// Forge is a message of the forge bridge's stream, trimmed to repository, branch, pull request and head. A pull
|
||||
// message says Action; a merged message carries the merge commit, the tree of every directory holding a
|
||||
// module.json and the repository's root tree; a refused message names the owed act it answers and the forge's own
|
||||
// words. A pull message may also carry the module directories its head changes (Dirs, in the order the forge said) and
|
||||
// DirsKnown, true when the forge could say them: known and empty is DirsKnown with no Dirs, not known is neither
|
||||
// (the forge's module_dirs and module_dirs_said, mesh-catalog modules/gitea/trunk.ts). Other messages carry neither.
|
||||
type Forge struct {
|
||||
V int `json:"v"`
|
||||
Kind string `json:"kind"`
|
||||
Repository string `json:"repository"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Action string `json:"action,omitempty"`
|
||||
Dirs []string `json:"dirs,omitempty"`
|
||||
DirsKnown bool `json:"dirs-known,omitempty"`
|
||||
Merge string `json:"merge,omitempty"`
|
||||
Trees map[string]string `json:"trees,omitempty"`
|
||||
Root string `json:"root,omitempty"`
|
||||
// Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message,
|
||||
// of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path
|
||||
// relative to it (novox/hq ADR 0465 clause 1). Older writers send neither.
|
||||
Manifests map[string]string `json:"manifests,omitempty"`
|
||||
Locks map[string]map[string]string `json:"locks,omitempty"`
|
||||
// Paths are, of each module directory whose manifest names paths on every artifact, each named path's git object
|
||||
// id at the same commit as its manifest, by path relative to the directory (novox/hq ADR 0499 clause 3). The
|
||||
// indexer names the paths; a message without them gives such a module none. Older writers send none.
|
||||
Paths map[string]map[string]string `json:"paths,omitempty"`
|
||||
// Packages are, of each module directory, the mesh packages its lock file names (MeshPackagesOf; novox/hq ADR 0573
|
||||
// §2): the indexer keeps them in the module's inputs and takes each one's latest published version into its
|
||||
// fingerprint. Absent for a directory that uses none, and from an older writer.
|
||||
Packages map[string][]string `json:"packages,omitempty"`
|
||||
Act string `json:"act,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Open []OpenPull `json:"open,omitempty"`
|
||||
// RetiredPlacements is the placement files a bridge before ADR 0567 carried (ADR 0549 §2, ADR 0559 §2): placement is
|
||||
// the seat mesh-placement's state now and no repository holds it (ADR 0567 §11), so a message carrying them is
|
||||
// refused, loudly, rather than read with them dropped.
|
||||
RetiredPlacements json.RawMessage `json:"placements,omitempty"`
|
||||
RetiredRead bool `json:"placements-read,omitempty"`
|
||||
// Predefined is the catalogue's predefined.json, exact, at the merge commit of a merged message (novox/hq ADR 0565
|
||||
// §2): what the indexer keeps as its state `predefined`. PredefinedRead says the bridge read the merge and it holds
|
||||
// no predefined.json. Carried on a merged message only; an older bridge sends neither, and the indexer keeps what it
|
||||
// held.
|
||||
Predefined json.RawMessage `json:"predefined,omitempty"`
|
||||
PredefinedRead bool `json:"predefined-read,omitempty"`
|
||||
// Group is the delivery group a merged message's pull request was merged in, and Members every approved member of
|
||||
// that group with its approved head, this one included (novox/hq ADR 0633 §4): the forge bridge sets both from the
|
||||
// gate's merges state, and the indexer gathers the group's merges until the group's end (ForgeGroupEnd). A merge
|
||||
// with no group carries neither, and an older bridge sends neither; an older reader ignores both.
|
||||
Group string `json:"group,omitempty"`
|
||||
Members []delivery.Head `json:"members,omitempty"`
|
||||
}
|
||||
|
||||
// checkGroup refuses a group on anything but a merged message, a group without its members or members without a
|
||||
// group, a member that is not a repository's pull request at a head, a member named twice, and members that leave out
|
||||
// the message's own pull request.
|
||||
func (f Forge) checkGroup() error {
|
||||
if f.Group == "" && len(f.Members) == 0 {
|
||||
return nil
|
||||
}
|
||||
if f.Kind != ForgeMerged {
|
||||
return fmt.Errorf("delivery: only a merged message names its delivery group, not a %s message", f.Kind)
|
||||
}
|
||||
if f.Group == "" || len(f.Members) == 0 {
|
||||
return fmt.Errorf("delivery: a merged message names its group and the group's members together, or neither")
|
||||
}
|
||||
self := false
|
||||
seen := map[string]bool{}
|
||||
for _, m := range f.Members {
|
||||
if err := checkRepository(m.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Number <= 0 || m.Head == "" {
|
||||
return fmt.Errorf("delivery: the member %s of the group %q needs its pull request number and head", m.Repository, f.Group)
|
||||
}
|
||||
k := fmt.Sprintf("%s#%d", m.Repository, m.Number)
|
||||
if seen[k] {
|
||||
return fmt.Errorf("delivery: the group %q names its member %s twice", f.Group, k)
|
||||
}
|
||||
seen[k] = true
|
||||
self = self || (delivery.SameRepository(m.Repository, f.Repository) && m.Number == f.Number)
|
||||
}
|
||||
if !self {
|
||||
return fmt.Errorf("delivery: the members of the group %q leave out the merged pull request %s#%d", f.Group, f.Repository, f.Number)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkPredefined refuses the predefined list on anything but a merged message, a list that is not one JSON object, and
|
||||
// a list said both present and absent.
|
||||
func (f Forge) checkPredefined() error {
|
||||
if len(f.Predefined) == 0 && !f.PredefinedRead {
|
||||
return nil
|
||||
}
|
||||
if f.Kind != ForgeMerged {
|
||||
return fmt.Errorf("delivery: only a merged message carries the predefined list, not a %s message", f.Kind)
|
||||
}
|
||||
if len(f.Predefined) > 0 && f.PredefinedRead {
|
||||
return fmt.Errorf("delivery: a merged message carries the predefined list and says the merge holds none (predefined-read)")
|
||||
}
|
||||
var list map[string]json.RawMessage
|
||||
if len(f.Predefined) > 0 && json.Unmarshal(f.Predefined, &list) != nil {
|
||||
return fmt.Errorf("delivery: the predefined list a merged message carries is not one JSON object")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f Forge) check() error {
|
||||
if err := checkForgeKind(f.Kind); err != nil {
|
||||
return err
|
||||
}
|
||||
if f.Kind == ForgeOpen {
|
||||
return f.checkOpen()
|
||||
}
|
||||
if len(f.Open) > 0 {
|
||||
return fmt.Errorf("delivery: only an open message names open pull requests")
|
||||
}
|
||||
if len(f.RetiredPlacements) > 0 || f.RetiredRead {
|
||||
return fmt.Errorf("delivery: a forge message carries placement files, which no repository holds since ADR 0567: " +
|
||||
"placement is the seat mesh-placement's state")
|
||||
}
|
||||
if err := f.checkPredefined(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.checkGroup(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkRepository(f.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
switch f.Kind {
|
||||
case ForgePull:
|
||||
if f.Number <= 0 {
|
||||
return fmt.Errorf("delivery: a pull message needs its pull request number")
|
||||
}
|
||||
if f.Head == "" {
|
||||
return fmt.Errorf("delivery: a pull message needs its head")
|
||||
}
|
||||
switch f.Action {
|
||||
case PullOpened, PullUpdated, PullClosed:
|
||||
default:
|
||||
return fmt.Errorf("delivery: the pull action %q is not opened, updated or closed", f.Action)
|
||||
}
|
||||
if len(f.Dirs) > 0 && !f.DirsKnown {
|
||||
return fmt.Errorf("delivery: a pull message with module directories must say it knows them (dirs-known)")
|
||||
}
|
||||
dirs := map[string]bool{}
|
||||
for _, d := range f.Dirs {
|
||||
dirs[d] = true
|
||||
}
|
||||
if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil {
|
||||
return err
|
||||
}
|
||||
case ForgeMerged:
|
||||
if len(f.Dirs) > 0 || f.DirsKnown {
|
||||
return fmt.Errorf("delivery: a merged message carries no module directories")
|
||||
}
|
||||
if f.Number <= 0 {
|
||||
return fmt.Errorf("delivery: a merged message needs its pull request number")
|
||||
}
|
||||
if f.Head == "" {
|
||||
return fmt.Errorf("delivery: a merged message needs its head")
|
||||
}
|
||||
if f.Merge == "" {
|
||||
return fmt.Errorf("delivery: a merged message needs its merge commit")
|
||||
}
|
||||
if f.Root == "" {
|
||||
return fmt.Errorf("delivery: a merged message needs the repository's root tree")
|
||||
}
|
||||
// No trees is a merge that leaves the repository with no module.json anywhere (a documents repository, or one
|
||||
// whose last module was removed): it is written all the same, so the indexer takes the merge in and drops the
|
||||
// repository's modules (novox/hq issue 1126). An older writer always sent at least one tree, so none is unambiguous;
|
||||
// such a message carries no manifests, locks or paths (checkInputs below: each names a directory of its trees).
|
||||
for dir, tree := range f.Trees {
|
||||
if tree == "" {
|
||||
return fmt.Errorf("delivery: a merged message's tree of the directory %q is empty", dir)
|
||||
}
|
||||
}
|
||||
if err := f.checkInputs(func(d string) bool { _, ok := f.Trees[d]; return ok }, "its trees"); err != nil {
|
||||
return err
|
||||
}
|
||||
case ForgeRefused:
|
||||
if len(f.Dirs) > 0 || f.DirsKnown {
|
||||
return fmt.Errorf("delivery: a refused message carries no module directories")
|
||||
}
|
||||
if len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 {
|
||||
return fmt.Errorf("delivery: a refused message carries no manifests, lock hashes or named paths")
|
||||
}
|
||||
if f.Act == "" {
|
||||
return fmt.Errorf("delivery: a refused message needs the act it answers")
|
||||
}
|
||||
if f.Words == "" {
|
||||
return fmt.Errorf("delivery: a refused message needs the forge's words")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkInputs refuses a manifest or a lock hash of a directory that known does not know (of is what it knows them
|
||||
// by), an empty manifest and an empty lock path or hash, naming the directory.
|
||||
func (f Forge) checkInputs(known func(dir string) bool, of string) error {
|
||||
for dir, m := range f.Manifests {
|
||||
if !known(dir) {
|
||||
return fmt.Errorf("delivery: the manifest of the directory %q is not of %s", dir, of)
|
||||
}
|
||||
if m == "" {
|
||||
return fmt.Errorf("delivery: the manifest of the directory %q is empty", dir)
|
||||
}
|
||||
}
|
||||
for dir, locks := range f.Locks {
|
||||
if !known(dir) {
|
||||
return fmt.Errorf("delivery: the lock hashes of the directory %q are not of %s", dir, of)
|
||||
}
|
||||
for path, hash := range locks {
|
||||
if path == "" || hash == "" {
|
||||
return fmt.Errorf("delivery: the lock %q of the directory %q has an empty path or hash", path, dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
for dir, paths := range f.Paths {
|
||||
if !known(dir) {
|
||||
return fmt.Errorf("delivery: the named paths of the directory %q are not of %s", dir, of)
|
||||
}
|
||||
for path, object := range paths {
|
||||
if path == "" || object == "" {
|
||||
return fmt.Errorf("delivery: the named path %q of the directory %q has an empty path or object", path, dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkOpen refuses an open message that names a repository, a branch, a head or anything of the other kinds, or
|
||||
// an open pull request that is not `<owner>/<name>` and a number.
|
||||
func (f Forge) checkOpen() error {
|
||||
if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown ||
|
||||
f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 || f.Root != "" || f.Act != "" || f.Words != "" || f.Group != "" || len(f.Members) > 0 {
|
||||
return fmt.Errorf("delivery: an open message carries only the open pull requests it names")
|
||||
}
|
||||
for _, p := range f.Open {
|
||||
if err := checkRepository(p.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.Number <= 0 {
|
||||
return fmt.Errorf("delivery: the open pull request of %s has no number", p.Repository)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeForge is f as JSON at Version; it refuses a message missing what its kind needs.
|
||||
func EncodeForge(f Forge) ([]byte, error) {
|
||||
if err := f.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.V = Version
|
||||
return json.Marshal(f)
|
||||
}
|
||||
|
||||
// DecodeForge reads a forge message; it refuses an unknown version (ErrUnknownVersion) and what EncodeForge refuses.
|
||||
func DecodeForge(data []byte) (Forge, error) {
|
||||
var f Forge
|
||||
if err := decode(data, &f); err != nil {
|
||||
return Forge{}, err
|
||||
}
|
||||
if err := f.check(); err != nil {
|
||||
return Forge{}, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// SameForge says whether two forge messages are the same value.
|
||||
func SameForge(a, b Forge) bool { return reflect.DeepEqual(a, b) }
|
||||
|
||||
var nameOK = regexp.MustCompile(`^[-/_=a-zA-Z0-9]+$`)
|
||||
|
||||
// checkName refuses a name that is empty or holds a dot, a wildcard, whitespace or a character a bus key refuses
|
||||
// (mesh-sdk delivery's rule for a name).
|
||||
func checkName(kind, name string) error {
|
||||
if name == "" {
|
||||
return fmt.Errorf("delivery: the %s name is empty", kind)
|
||||
}
|
||||
if !nameOK.MatchString(name) {
|
||||
return fmt.Errorf("delivery: the %s name %q holds a dot, a wildcard, whitespace or a character a key refuses", kind, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// decode reads a value of Version, refusing another or none with delivery.ErrUnknownVersion, as mesh-sdk delivery's
|
||||
// values are read.
|
||||
func decode(data []byte, into any) error {
|
||||
var probe struct {
|
||||
V *int `json:"v"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &probe); err != nil {
|
||||
return fmt.Errorf("delivery: value is not JSON: %w", err)
|
||||
}
|
||||
if probe.V == nil {
|
||||
return fmt.Errorf("%w: no version in the value, this reader reads %d", delivery.ErrUnknownVersion, Version)
|
||||
}
|
||||
if *probe.V != Version {
|
||||
return fmt.Errorf("%w: the value is version %d, this reader reads %d", delivery.ErrUnknownVersion, *probe.V, Version)
|
||||
}
|
||||
if err := json.Unmarshal(data, into); err != nil {
|
||||
return fmt.Errorf("delivery: value does not fit its shape: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package forgemsg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const oddManifest = "{\n\t\"module\": \"a\",\n \"z\": 1, \"a\": [ \"x\",\"y\" ],\n \"why\": \"é — <b> & \\n\"\n}\n\n"
|
||||
|
||||
func sampleForges() []Forge {
|
||||
return []Forge{
|
||||
{Kind: ForgePull, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 36, Head: "aaaa", Action: "opened"},
|
||||
{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc",
|
||||
Trees: map[string]string{"modules/a": "t1"}, Root: "t0"},
|
||||
{Kind: ForgeRefused, Repository: "novox/mesh-sdk", Act: "novox/mesh-sdk.36.aaaa.view", Words: "403 forbidden"},
|
||||
{Kind: ForgeOpen, Open: []OpenPull{{Repository: "novox/mesh-sdk", Number: 36}, {Repository: "novox/mesh-control", Number: 7}}},
|
||||
// 1585/T1: a group member's merged message names its group and every approved member (ADR 0633 §4).
|
||||
{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 37, Head: "dddd", Merge: "eeee", Root: "t0",
|
||||
Group: "feat/take-in", Members: []delivery.Head{{Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}, {Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}}},
|
||||
}
|
||||
}
|
||||
|
||||
func mergedWithInputs() Forge {
|
||||
return Forge{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc", Root: "t0",
|
||||
Trees: map[string]string{"a": "t1", "b": "t2"},
|
||||
Manifests: map[string]string{"a": oddManifest},
|
||||
Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("ab", 32)}}}
|
||||
}
|
||||
|
||||
// TestForge holds the forge bridge's contract, moved whole from mesh-sdk go/delivery with its tests (1200/AC1, AC3,
|
||||
// AC4; 1279/AC5; 1442/AC1-AC4; 1585/T1).
|
||||
func TestForge(t *testing.T) {
|
||||
t.Run("events", func(t *testing.T) {
|
||||
ev, err := ForgeEvent("novox/mesh-sdk", ForgeMerged)
|
||||
if err != nil || ev != "forge.novox.mesh-sdk.merged" {
|
||||
t.Fatalf("forge event = %q, %v", ev, err)
|
||||
}
|
||||
if repo, kind, err := ParseForgeEvent(ev); err != nil || repo != "novox/mesh-sdk" || kind != ForgeMerged {
|
||||
t.Fatalf("parse forge event = %q %q %v", repo, kind, err)
|
||||
}
|
||||
for _, bad := range []struct{ repo, kind string }{
|
||||
{"", "pull"}, {"novox", "pull"}, {"a/b/c", "pull"}, {"/b", "pull"}, {"a/", "pull"},
|
||||
{"a.b/c", "pull"}, {"a/b c", "pull"}, {"a/b", "closed"}, {"a/b", ""}, {"a/b", ForgeOpen},
|
||||
} {
|
||||
if _, err := ForgeEvent(bad.repo, bad.kind); err == nil {
|
||||
t.Errorf("forge event %q %q accepted", bad.repo, bad.kind)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "forge.a.b", "forge.a.b.closed", "x.a.b.pull", "forge.a.b.pull.x", "forge..b.pull", ForgeGroupEndEvent} {
|
||||
if _, _, err := ParseForgeEvent(bad); err == nil {
|
||||
t.Errorf("parse forge event %q accepted", bad)
|
||||
}
|
||||
}
|
||||
if repo, kind, err := ParseForgeEvent(OpenEvent()); OpenEvent() != "forge.open" || err != nil || repo != "" || kind != ForgeOpen {
|
||||
t.Errorf("open event %q = %q %q %v", OpenEvent(), repo, kind, err)
|
||||
}
|
||||
})
|
||||
t.Run("round trip", func(t *testing.T) {
|
||||
fs := append(sampleForges(), mergedWithInputs())
|
||||
for _, f := range fs {
|
||||
data, err := EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", f.Kind, err)
|
||||
}
|
||||
var probe map[string]any
|
||||
if err := json.Unmarshal(data, &probe); err != nil || probe["v"] != float64(1) {
|
||||
t.Fatalf("%s: json %s does not carry v:1", f.Kind, data)
|
||||
}
|
||||
f.V = Version
|
||||
if got, err := DecodeForge(data); err != nil || !SameForge(got, f) {
|
||||
t.Errorf("forge %s round trip: %+v %v", f.Kind, got, err)
|
||||
}
|
||||
}
|
||||
// An empty list is an open message too: nothing is open.
|
||||
if data, err := EncodeForge(Forge{Kind: ForgeOpen, Open: []OpenPull{}}); err != nil {
|
||||
t.Errorf("empty open: %v", err)
|
||||
} else if f, err := DecodeForge(data); err != nil || f.Kind != ForgeOpen || len(f.Open) != 0 {
|
||||
t.Errorf("empty open round trip: %+v %v", f, err)
|
||||
}
|
||||
ge := ForgeGroupEnd{Group: "feat/take-in", Members: []GroupMerge{{Repository: "novox/mesh-sdk", Number: 37, Merge: "eeee"}}}
|
||||
data, err := EncodeForgeGroupEnd(ge)
|
||||
ge.V = Version
|
||||
if got, err2 := DecodeForgeGroupEnd(data); err != nil || err2 != nil || !SameForgeGroupEnd(got, ge) {
|
||||
t.Errorf("group end round trip: %+v %v %v", got, err, err2)
|
||||
}
|
||||
// An older bridge's message reads, and is written back byte for byte.
|
||||
old := `{"v":1,"kind":"merged","repository":"novox/mesh-sdk","number":36,"head":"aaaa","merge":"cccc","trees":{"a":"t1"},"root":"t0"}`
|
||||
if f, err := DecodeForge([]byte(old)); err != nil || f.Manifests != nil {
|
||||
t.Fatalf("%+v, %v", f, err)
|
||||
} else if data, err := EncodeForge(f); err != nil || string(data) != old {
|
||||
t.Errorf("%s, %v, want exactly %s", data, err, old)
|
||||
}
|
||||
// A merge leaving the repository with no module.json is a message too (issue 1126).
|
||||
f := sampleForges()[1]
|
||||
f.Trees = nil
|
||||
if data, err := EncodeForge(f); err != nil {
|
||||
t.Errorf("a merged message with no trees is refused: %v", err)
|
||||
} else if got, err := DecodeForge(data); err != nil || len(got.Trees) != 0 {
|
||||
t.Errorf("a merged message with no trees comes back as %+v, %v", got, err)
|
||||
}
|
||||
for name, dec := range map[string]func([]byte) error{
|
||||
"forge": func(b []byte) error { _, err := DecodeForge(b); return err },
|
||||
"group end": func(b []byte) error { _, err := DecodeForgeGroupEnd(b); return err },
|
||||
} {
|
||||
for _, in := range []string{`{"group":"g"}`, `{"v":2,"group":"g"}`} {
|
||||
if err := dec([]byte(in)); !errors.Is(err, delivery.ErrUnknownVersion) {
|
||||
t.Errorf("%s decode of %s = %v, want delivery.ErrUnknownVersion", name, in, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("refused", func(t *testing.T) {
|
||||
at := func(i int, mut func(*Forge)) Forge { f := sampleForges()[i]; mut(&f); return f }
|
||||
in := func(mut func(*Forge)) Forge { f := mergedWithInputs(); mut(&f); return f }
|
||||
pullIn := func(mut func(*Forge)) Forge {
|
||||
f := Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened,
|
||||
Dirs: []string{"a"}, DirsKnown: true, Manifests: map[string]string{"a": oddManifest},
|
||||
Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("0", 64)}}}
|
||||
mut(&f)
|
||||
return f
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name, want string
|
||||
f Forge
|
||||
}{
|
||||
{"merged without merge commit", "merge", at(1, func(f *Forge) { f.Merge = "" })},
|
||||
{"merged without root", "root", at(1, func(f *Forge) { f.Root = "" })},
|
||||
{"merged with a manifest and no trees", "modules/a", at(1, func(f *Forge) { f.Trees = nil; f.Manifests = map[string]string{"modules/a": "{}"} })},
|
||||
{"merged with an empty tree", "modules/a", at(1, func(f *Forge) { f.Trees = map[string]string{"modules/a": ""} })},
|
||||
{"pull with an unknown action", "reopened", at(0, func(f *Forge) { f.Action = "reopened" })},
|
||||
{"pull without head", "head", at(0, func(f *Forge) { f.Head = "" })},
|
||||
{"refused without words", "words", at(2, func(f *Forge) { f.Words = "" })},
|
||||
{"refused without act", "act", at(2, func(f *Forge) { f.Act = "" })},
|
||||
{"unknown kind", "closed", Forge{Kind: "closed", Repository: "a/b"}},
|
||||
{"bad repository", "nope", Forge{Kind: ForgePull, Repository: "nope", Number: 1, Head: "h", Action: "opened"}},
|
||||
{"open with a repository", "only the open", Forge{Kind: ForgeOpen, Repository: "a/b", Open: []OpenPull{{Repository: "a/b", Number: 1}}}},
|
||||
{"open with a bad entry", "", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a", Number: 1}}}},
|
||||
{"open with a zero number", "no number", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a/b"}}}},
|
||||
{"pull naming open pull requests", "only an open message", Forge{Kind: ForgePull, Repository: "a/b", Number: 1, Head: "h", Action: "updated", Open: []OpenPull{{Repository: "a/b", Number: 1}}}},
|
||||
{"manifest outside trees", `"zz"`, in(func(f *Forge) { f.Manifests["zz"] = "{}" })},
|
||||
{"locks outside trees", `"yy"`, in(func(f *Forge) { f.Locks["yy"] = map[string]string{"go.sum": "h"} })},
|
||||
{"empty manifest", `"b"`, in(func(f *Forge) { f.Manifests["b"] = "" })},
|
||||
{"empty lock hash", `"a"`, in(func(f *Forge) { f.Locks["a"]["go.sum"] = "" })},
|
||||
{"named paths outside trees", `"other"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"other": {"src": "o"}} })},
|
||||
{"a named path with no object", `"src"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"a": {"src": ""}} })},
|
||||
{"placement files", "mesh-placement", in(func(f *Forge) { f.RetiredPlacements = []byte(`{"anchor":"{}"}`) })},
|
||||
{"placements read", "mesh-placement", in(func(f *Forge) { f.RetiredRead = true })},
|
||||
{"predefined not an object", "one JSON object", in(func(f *Forge) { f.Predefined = []byte(`["domain"]`) })},
|
||||
{"predefined both", "says the merge holds none", in(func(f *Forge) { f.Predefined = []byte(`{}`); f.PredefinedRead = true })},
|
||||
{"pull with the predefined list", "only a merged message", Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened, PredefinedRead: true}},
|
||||
{"pull manifest outside dirs", `"b"`, pullIn(func(f *Forge) { f.Manifests["b"] = "{}" })},
|
||||
{"pull locks outside dirs", `"c"`, pullIn(func(f *Forge) { f.Locks["c"] = map[string]string{"go.sum": "h"} })},
|
||||
{"refused with manifests", "no manifests", Forge{Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Manifests: map[string]string{"a": "{}"}}},
|
||||
{"open with locks", "only the open", Forge{Kind: ForgeOpen, Locks: map[string]map[string]string{"a": {"go.sum": "h"}}}},
|
||||
// 1585/T1: a group only on a merged message, with its members, the message's own pull request among them.
|
||||
{"pull with a group", "only a merged message names its delivery group", at(0, func(f *Forge) {
|
||||
f.Group, f.Members = "g", []delivery.Head{{Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa"}}
|
||||
})},
|
||||
{"merged with a group and no members", "together", at(4, func(f *Forge) { f.Members = nil })},
|
||||
{"merged with members and no group", "together", at(4, func(f *Forge) { f.Group = "" })},
|
||||
{"merged with members leaving itself out", "leave out", at(4, func(f *Forge) { f.Members = f.Members[1:] })},
|
||||
{"merged with a member twice", "twice", at(4, func(f *Forge) { f.Members = append(f.Members, f.Members[0]) })},
|
||||
{"open with a group", "only the open pull requests", Forge{Kind: ForgeOpen, Group: "g"}},
|
||||
} {
|
||||
if _, err := EncodeForge(c.f); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want)
|
||||
}
|
||||
}
|
||||
// A member's repository in another case still names the merged pull request (10c91fba).
|
||||
f := sampleForges()[4]
|
||||
f.Members[0].Repository = "Novox/Mesh-SDK"
|
||||
if _, err := EncodeForge(f); err != nil {
|
||||
t.Errorf("a member in another case: %v", err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name, want string
|
||||
e ForgeGroupEnd
|
||||
}{
|
||||
{"group end without members", "no member", ForgeGroupEnd{Group: "g"}},
|
||||
{"group end without a group", "names its group", ForgeGroupEnd{Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}}}},
|
||||
{"group end with a member without its merge", "merge commit", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1}}}},
|
||||
{"group end with a member twice", "twice", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}, {Repository: "a/b", Number: 1, Merge: "m"}}}},
|
||||
} {
|
||||
if _, err := EncodeForgeGroupEnd(c.e); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
|
||||
@@ -0,0 +1,14 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
@@ -0,0 +1,78 @@
|
||||
package forgemsg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// ForgeGroupEndEvent is the name of a delivery group's end on the forge bridge's stream (novox/hq ADR 0633 §4): the
|
||||
// bridge publishes it once every approved member of the group is merged, after the last member's merged message. It
|
||||
// has three parts, so no repository's forge event (`forge.<owner>.<name>.<kind>`) is ever this name.
|
||||
const ForgeGroupEndEvent = "forge.group.merged"
|
||||
|
||||
// GroupMerge is one merged member of a delivery group: the repository as `<owner>/<name>`, its pull request and the
|
||||
// merge commit the forge made of it.
|
||||
type GroupMerge struct {
|
||||
Repository string `json:"repository"`
|
||||
Number int `json:"number"`
|
||||
Merge string `json:"merge"`
|
||||
}
|
||||
|
||||
// ForgeGroupEnd is the body of ForgeGroupEndEvent: the group and every member merged in it, with its merge commit.
|
||||
// The indexer takes the members it holds in as one source revision once it hears it (ADR 0633 §4).
|
||||
type ForgeGroupEnd struct {
|
||||
V int `json:"v"`
|
||||
Group string `json:"group"`
|
||||
Members []GroupMerge `json:"members"`
|
||||
}
|
||||
|
||||
func (e ForgeGroupEnd) check() error {
|
||||
if e.Group == "" {
|
||||
return fmt.Errorf("delivery: a group's end names its group")
|
||||
}
|
||||
if len(e.Members) == 0 {
|
||||
return fmt.Errorf("delivery: the end of the group %q names no member merged in it", e.Group)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range e.Members {
|
||||
if err := checkRepository(m.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Number <= 0 || m.Merge == "" {
|
||||
return fmt.Errorf("delivery: the member %s of the group %q's end needs its pull request number and merge commit", m.Repository, e.Group)
|
||||
}
|
||||
k := fmt.Sprintf("%s#%d", m.Repository, m.Number)
|
||||
if seen[k] {
|
||||
return fmt.Errorf("delivery: the end of the group %q names its member %s twice", e.Group, k)
|
||||
}
|
||||
seen[k] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeForgeGroupEnd is e as JSON at Version; it refuses an end with no group or no member, and a member without its
|
||||
// pull request or merge commit.
|
||||
func EncodeForgeGroupEnd(e ForgeGroupEnd) ([]byte, error) {
|
||||
if err := e.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.V = Version
|
||||
return json.Marshal(e)
|
||||
}
|
||||
|
||||
// DecodeForgeGroupEnd reads a group's end; it refuses an unknown version (ErrUnknownVersion) and what
|
||||
// EncodeForgeGroupEnd refuses.
|
||||
func DecodeForgeGroupEnd(data []byte) (ForgeGroupEnd, error) {
|
||||
var e ForgeGroupEnd
|
||||
if err := decode(data, &e); err != nil {
|
||||
return ForgeGroupEnd{}, err
|
||||
}
|
||||
if err := e.check(); err != nil {
|
||||
return ForgeGroupEnd{}, err
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// SameForgeGroupEnd says whether two group ends are the same value.
|
||||
func SameForgeGroupEnd(a, b ForgeGroupEnd) bool { return reflect.DeepEqual(a, b) }
|
||||
@@ -3,12 +3,13 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
@@ -20,3 +21,5 @@ require (
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ./forgemsg
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -145,11 +146,11 @@ func MergeActs(value []byte) ([]Act, error) {
|
||||
// RefusedMessage is the refused message the bridge writes on FORGE when the forge refuses an act: the act, the pull
|
||||
// request and head it was on, and the forge's own words.
|
||||
func RefusedMessage(a Act, words string) (Message, error) {
|
||||
event, err := delivery.ForgeEvent(a.Repository, delivery.ForgeRefused)
|
||||
event, err := forgemsg.ForgeEvent(a.Repository, forgemsg.ForgeRefused)
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeRefused, Repository: a.Repository, Number: a.Number,
|
||||
body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeRefused, Repository: a.Repository, Number: a.Number,
|
||||
Head: a.Head, Act: a.Name, Words: words})
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -27,7 +28,7 @@ func owedValue(t *testing.T, repository string, number int, act string, o delive
|
||||
return key, b
|
||||
}
|
||||
|
||||
func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1
|
||||
func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1, K1
|
||||
key, v := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate",
|
||||
delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "pending", Text: "waiting", Target: "https://forge.example/novox/mesh-delivery/pulls/12"})
|
||||
a, err := OwedAct(key, v)
|
||||
@@ -42,6 +43,19 @@ func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1
|
||||
if a.Repository != "novox/mesh-delivery" || a.Number != 12 || a.Head != headA || a.Name != "status=mesh/merge-gate" {
|
||||
t.Errorf("act = %+v", a)
|
||||
}
|
||||
// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no
|
||||
// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge
|
||||
// performs no mesh/check act.
|
||||
ck, cv := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"})
|
||||
if a, err := OwedAct(ck, cv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
|
||||
t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err)
|
||||
}
|
||||
// Nor through the gate's own owed bucket: the context is refused whichever key carries it.
|
||||
gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check",
|
||||
delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"})
|
||||
if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
|
||||
t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
// checkerKV is the status the checker owes, as it writes it (mesh-sdk go/delivery CheckerOwedKey, EncodeCheckerOwed).
|
||||
@@ -58,23 +72,6 @@ func checkerKV(t *testing.T, o delivery.Owed) (string, []byte) {
|
||||
return key, v
|
||||
}
|
||||
|
||||
// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no
|
||||
// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge
|
||||
// performs no mesh/check act.
|
||||
func TestOwedMeshCheckIsNoToolCall(t *testing.T) { // K1
|
||||
key, v := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"})
|
||||
a, err := OwedAct(key, v)
|
||||
if err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
|
||||
t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err)
|
||||
}
|
||||
// Nor through the gate's own owed bucket: the context is refused whichever key carries it.
|
||||
gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check",
|
||||
delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"})
|
||||
if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
|
||||
t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedViewNoteAndCommentCalls(t *testing.T) { // 1341/AC2
|
||||
cases := []struct {
|
||||
act string
|
||||
@@ -111,18 +108,12 @@ func TestOwedActWithoutAPullRequestNeedsOneOnlyWhereTheToolDoes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedActOfADottedRepositoryIsUnescaped(t *testing.T) {
|
||||
key, v := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"})
|
||||
a, err := OwedAct(key, v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" {
|
||||
t.Errorf("act = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedActThatDisagreesWithItsKeyIsRefused(t *testing.T) {
|
||||
// A dotted repository's key is unescaped, and its act names the repository whole.
|
||||
dk, dv := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"})
|
||||
if a, err := OwedAct(dk, dv); err != nil || a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" {
|
||||
t.Errorf("a dotted repository's act = %+v, %v", a, err)
|
||||
}
|
||||
key, _ := owedValue(t, "novox/mesh-delivery", 12, "view", delivery.Owed{Act: delivery.OwedView, Text: "v"})
|
||||
v, _ := delivery.EncodeOwed(delivery.Owed{Act: delivery.OwedComment, Text: "v"})
|
||||
if _, err := OwedAct(key, v); err == nil {
|
||||
@@ -166,13 +157,11 @@ func TestMergeActsHandOnlyTheMemberHandedWithEveryApprovedHead(t *testing.T) { /
|
||||
if acts[0].Name != "merge" || acts[0].Number != 12 || acts[0].Head != headA {
|
||||
t.Errorf("act = %+v", acts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeActsOfAMergeWithoutApprovedHeadsAreRefused(t *testing.T) {
|
||||
v := mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{
|
||||
// A merge with no approved heads is refused: the forge's tool would refuse it with nothing to say which.
|
||||
v = mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, State: delivery.MergeMerging}}})
|
||||
if _, err := MergeActs(v); err == nil {
|
||||
t.Error("a merge with no approved heads was taken: the forge's tool would refuse it with nothing to say which")
|
||||
t.Error("a merge with no approved heads was taken")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,13 +174,13 @@ func TestRefusedMessageNamesTheActAndTheForgesWords(t *testing.T) { // 1341/AC3,
|
||||
if m.Event != "forge.novox.mesh-delivery.refused" {
|
||||
t.Errorf("event = %q", m.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(m.Data)
|
||||
f, err := forgemsg.DecodeForge(m.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA,
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA,
|
||||
Act: "merge", Words: "novox/mesh-delivery#12 is not merged: its head moved"}
|
||||
if !delivery.SameForge(f, want) {
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// ErrUnreadable is wrapped by every error that says an event cannot be turned into a message; it is never
|
||||
@@ -29,9 +29,6 @@ var ErrNotHandled = errors.New("event not handled")
|
||||
// heard again later.
|
||||
var ErrRetry = errors.New("the forge could not say")
|
||||
|
||||
// ErrNoModules says a merge into the trunk left a repository with no module.json anywhere: no merged message.
|
||||
var ErrNoModules = errors.New("no module.json in the merge")
|
||||
|
||||
// Trunk is the branch a merged message is written for.
|
||||
const Trunk = "main"
|
||||
|
||||
@@ -100,7 +97,9 @@ func unreadable(subject, why string, args ...any) error {
|
||||
// Convert turns one event of the forge module, by its bus subject and body, into a message. An event it cannot read
|
||||
// is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into
|
||||
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
|
||||
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
|
||||
// pull request left its group. A forge that cannot say is ErrRetry. A merge that leaves no module.json anywhere is a merged message with no
|
||||
// trees all the same (issue 1126): the indexer takes it in and drops the repository's modules, and the view reads the
|
||||
// repository as reached at it.
|
||||
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
|
||||
return ConvertKept(subject, data, trees, files, nil)
|
||||
}
|
||||
@@ -111,11 +110,11 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
|
||||
var action string
|
||||
switch subject {
|
||||
case UpdatedSubject:
|
||||
action = delivery.PullUpdated
|
||||
action = forgemsg.PullUpdated
|
||||
case ClosedSubject:
|
||||
action = delivery.PullClosed
|
||||
action = forgemsg.PullClosed
|
||||
case MergedSubject:
|
||||
action = delivery.PullClosed // unless it merged into the trunk, below
|
||||
action = forgemsg.PullClosed // unless it merged into the trunk, below
|
||||
default:
|
||||
return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject)
|
||||
}
|
||||
@@ -137,29 +136,29 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
|
||||
return merged(subject, p, trees, files, kept)
|
||||
}
|
||||
said := ""
|
||||
if action == delivery.PullUpdated && p.Base != "" && p.Base != Trunk {
|
||||
if action == forgemsg.PullUpdated && p.Base != "" && p.Base != Trunk {
|
||||
// The checker judges pull requests into main only, and source.main names main's head only (novox/hq ADR 0484 §11): a
|
||||
// pull request into another base is no candidate, as it is none at the start (OpenPulls) and at its merge. It is
|
||||
// written closed, so a candidate an older head made is taken out, and said (issue 1422).
|
||||
action = delivery.PullClosed
|
||||
action = forgemsg.PullClosed
|
||||
said = fmt.Sprintf("%s/%s#%d at %s is into %s, not %s: written closed, so it is no candidate; the checker judges pull requests into %s only (novox/hq ADR 0484 §11)",
|
||||
p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base, Trunk, Trunk)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: action}
|
||||
// The directories are the forge's word only when it said it could read them (module_dirs_said).
|
||||
if action == delivery.PullUpdated && p.ModuleDirsSaid {
|
||||
if action == forgemsg.PullUpdated && p.ModuleDirsSaid {
|
||||
f.Dirs, f.DirsKnown = p.ModuleDirs, true
|
||||
var err error
|
||||
if f.Manifests, f.Locks, _, err = inputsRead(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
|
||||
}
|
||||
}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
body, err := delivery.EncodeForge(f)
|
||||
body, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
@@ -294,9 +293,6 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the trees of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
|
||||
}
|
||||
if len(dirs) == 0 {
|
||||
return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s", ErrNoModules, subject, repository, p.MergeSHA)
|
||||
}
|
||||
names := make([]string, 0, len(dirs))
|
||||
for dir := range dirs {
|
||||
names = append(names, dir)
|
||||
@@ -317,11 +313,11 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
|
||||
if len(locks) == 0 {
|
||||
locks = nil
|
||||
}
|
||||
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
|
||||
event, err := forgemsg.ForgeEvent(repository, forgemsg.ForgeMerged)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
f := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
|
||||
if repository == CatalogRepository {
|
||||
// The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config.
|
||||
@@ -335,11 +331,15 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
|
||||
f.Predefined = json.RawMessage(raw)
|
||||
}
|
||||
}
|
||||
body, err := delivery.EncodeForge(f)
|
||||
body, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
return Message{Event: event, Data: body}, nil
|
||||
msg := Message{Event: event, Data: body}
|
||||
if len(dirs) == 0 {
|
||||
msg.Said = fmt.Sprintf("%s at the merge commit %s holds no module.json: a merged message with no module directories", repository, p.MergeSHA)
|
||||
}
|
||||
return msg, nil
|
||||
}
|
||||
|
||||
// openPull is an entry of gitea_open_pulls's answer (mesh-catalog modules/gitea/client.ts:369-381 openPulls, with the
|
||||
@@ -373,12 +373,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
|
||||
skipped = append(skipped, name+": it names no owner, repo, number, branch or 40-hex head")
|
||||
continue
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: delivery.PullUpdated}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: forgemsg.PullUpdated}
|
||||
event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
|
||||
var body []byte
|
||||
if err == nil {
|
||||
body, err = delivery.EncodeForge(f)
|
||||
body, err = forgemsg.EncodeForge(f)
|
||||
}
|
||||
if err != nil {
|
||||
skipped = append(skipped, name+": "+err.Error())
|
||||
@@ -392,12 +392,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
|
||||
// Track says what a message the bridge wrote changes in the head it keeps for each pull request (the bucket heads,
|
||||
// novox/hq ADR 0443 §5): the key `<owner>.<name>.<number>`, and the head to keep under it, or "" to take the key away
|
||||
// (a closed pull request, a merged one). ok is false for a message that keeps no head.
|
||||
func Track(f delivery.Forge) (key, head string, ok bool) {
|
||||
func Track(f forgemsg.Forge) (key, head string, ok bool) {
|
||||
key = strings.ReplaceAll(f.Repository, "/", ".") + "." + strconv.Itoa(f.Number)
|
||||
switch {
|
||||
case f.Kind == delivery.ForgePull && f.Action != delivery.PullClosed:
|
||||
case f.Kind == forgemsg.ForgePull && f.Action != forgemsg.PullClosed:
|
||||
return key, f.Head, true
|
||||
case f.Kind == delivery.ForgePull || f.Kind == delivery.ForgeMerged:
|
||||
case f.Kind == forgemsg.ForgePull || f.Kind == forgemsg.ForgeMerged:
|
||||
return key, "", true
|
||||
}
|
||||
return "", "", false
|
||||
@@ -413,22 +413,22 @@ func OpenMessage(answer []byte) (Message, error) {
|
||||
if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil {
|
||||
return Message{}, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err)
|
||||
}
|
||||
open := []delivery.OpenPull{}
|
||||
open := []forgemsg.OpenPull{}
|
||||
for _, p := range *a.Pulls {
|
||||
if p.Base != Trunk || p.Owner == "" || p.Repo == "" || p.Number <= 0 {
|
||||
continue
|
||||
}
|
||||
open = append(open, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
|
||||
open = append(open, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
|
||||
}
|
||||
slices.SortFunc(open, func(a, b delivery.OpenPull) int {
|
||||
slices.SortFunc(open, func(a, b forgemsg.OpenPull) int {
|
||||
if c := strings.Compare(a.Repository, b.Repository); c != 0 {
|
||||
return c
|
||||
}
|
||||
return a.Number - b.Number
|
||||
})
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeOpen, Open: open})
|
||||
body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeOpen, Open: open})
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("the open pull requests cannot be said: %w", err)
|
||||
}
|
||||
return Message{Event: delivery.OpenEvent(), Data: body}, nil
|
||||
return Message{Event: forgemsg.OpenEvent(), Data: body}, nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -54,13 +55,13 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion;
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" {
|
||||
t.Errorf("event = %q", msg.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
}
|
||||
@@ -70,7 +71,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ := delivery.DecodeForge(msg.Data)
|
||||
f, _ := forgemsg.DecodeForge(msg.Data)
|
||||
if f.DirsKnown || len(f.Dirs) != 0 {
|
||||
t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
@@ -79,7 +80,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ = delivery.DecodeForge(msg.Data)
|
||||
f, _ = forgemsg.DecodeForge(msg.Data)
|
||||
if !f.DirsKnown || len(f.Dirs) != 0 {
|
||||
t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
@@ -88,7 +89,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ = delivery.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 {
|
||||
if f, _ = forgemsg.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 {
|
||||
t.Errorf("dirs %v known %v without module_dirs_said", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
}
|
||||
@@ -101,13 +102,13 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" {
|
||||
t.Errorf("event = %q", msg.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullClosed}
|
||||
if !delivery.SameForge(f, want) {
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: forgemsg.PullClosed}
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
// Issue 1422, ADR 0484 §11: a head pushed to a pull request into another base is written closed, read nothing for, and said;
|
||||
@@ -116,12 +117,12 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
|
||||
if f, err = forgemsg.DecodeForge(msg.Data); err != nil || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
|
||||
t.Errorf("an update into nox-mesh is %+v (%v), said %q; want it closed and said", f, err, msg.Said)
|
||||
}
|
||||
for _, base := range []any{"main", nil} {
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"base": base}), nil, anyFiles)
|
||||
if f, _ = delivery.DecodeForge(msg.Data); err != nil || f.Action != delivery.PullUpdated || msg.Said != "" {
|
||||
if f, _ = forgemsg.DecodeForge(msg.Data); err != nil || f.Action != forgemsg.PullUpdated || msg.Said != "" {
|
||||
t.Errorf("an update with base %v is %+v (%v), said %q; want it updated", base, f, err, msg.Said)
|
||||
}
|
||||
}
|
||||
@@ -170,12 +171,18 @@ func mustDrop(body []byte, key string) []byte {
|
||||
return b
|
||||
}
|
||||
|
||||
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235
|
||||
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236
|
||||
for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} {
|
||||
if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) {
|
||||
t.Errorf("%s: err = %v, want ErrNotHandled", s, err)
|
||||
}
|
||||
}
|
||||
// 1236: a merged event without a full merge commit is unreadable, and the forge is not asked.
|
||||
var calls []call
|
||||
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
|
||||
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
|
||||
t.Errorf("merged without a merge commit: err = %v calls = %v", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
@@ -222,13 +229,13 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
|
||||
if msg.Event != "forge.novox.mesh-delivery.merged" {
|
||||
t.Errorf("event = %q", msg.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
if len(calls) != 1 || calls[0] != (call{"novox", "mesh-delivery", merge}) {
|
||||
@@ -242,13 +249,13 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullClosed}
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" || !delivery.SameForge(f, want) {
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: forgemsg.PullClosed}
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" || !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("event %q message %+v, want a closed pull message", msg.Event, f)
|
||||
}
|
||||
if len(calls) != 0 {
|
||||
@@ -269,19 +276,19 @@ func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the co
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (notes: a repository with no module.json)
|
||||
// 1126 (note 94): a merge that leaves no module.json anywhere (hq, or a repository whose last module was removed) is a merged
|
||||
// message with no trees, so the indexer takes it in and the repository's reach is not held for ever.
|
||||
func TestNoModuleJSONAnywhereWritesAMergedMessageWithNoTrees(t *testing.T) {
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil), anyFiles)
|
||||
if !errors.Is(err, ErrNoModules) || msg.Data != nil {
|
||||
t.Errorf("err = %v data = %s", err, msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236
|
||||
var calls []call
|
||||
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
|
||||
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
|
||||
t.Errorf("err = %v calls = %v", err, calls)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Merge: merge, Root: "r0"}
|
||||
if err != nil || msg.Event != "forge.novox.mesh-delivery.merged" || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "no module.json") {
|
||||
t.Errorf("event %q message %+v said %q, %v; want a merged message with no trees", msg.Event, f, msg.Said, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,12 +310,12 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/
|
||||
if err != nil || len(skipped) != 0 || len(msgs) != 2 {
|
||||
t.Fatalf("msgs %d skipped %v err %v", len(msgs), skipped, err)
|
||||
}
|
||||
for i, want := range []delivery.Forge{
|
||||
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated},
|
||||
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated},
|
||||
for i, want := range []forgemsg.Forge{
|
||||
{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated},
|
||||
{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated},
|
||||
} {
|
||||
f, err := delivery.DecodeForge(msgs[i].Data)
|
||||
if err != nil || !delivery.SameForge(f, want) {
|
||||
f, err := forgemsg.DecodeForge(msgs[i].Data)
|
||||
if err != nil || !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message %d = %+v err %v, want %+v", i, f, err, want)
|
||||
}
|
||||
if f.DirsKnown {
|
||||
@@ -320,14 +327,11 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenPullIntoAnotherBranchGetsNoMessage(t *testing.T) { // 1237/AC4
|
||||
msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head)))
|
||||
if err != nil || len(msgs) != 0 || len(skipped) != 0 {
|
||||
t.Errorf("msgs %d skipped %v err %v, want none", len(msgs), skipped, err)
|
||||
func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237, 1237/AC4
|
||||
// 1237/AC4: an open pull request into another branch gets no message.
|
||||
if msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head))); err != nil || len(msgs) != 0 || len(skipped) != 0 {
|
||||
t.Errorf("into another branch: msgs %d skipped %v err %v, want none", len(msgs), skipped, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237
|
||||
msgs, skipped, err := OpenPulls(openAnswer(openEntry("a", 1, "main", "b", "short"), openEntry("mesh-delivery", 12, "main", "feat/x", head)))
|
||||
if err != nil || len(msgs) != 1 || len(skipped) != 1 || !strings.Contains(skipped[0], "novox/a#1") {
|
||||
t.Errorf("msgs %d skipped %v err %v", len(msgs), skipped, err)
|
||||
@@ -347,15 +351,15 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6
|
||||
if err != nil || m.Event != "forge.open" {
|
||||
t.Fatalf("message %+v err %v", m, err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(m.Data)
|
||||
f, err := forgemsg.DecodeForge(m.Data)
|
||||
// Every one into the trunk, whether its head reads or not (it is open); none into another branch; in a fixed order.
|
||||
want := []delivery.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
if err != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] {
|
||||
want := []forgemsg.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
if err != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] {
|
||||
t.Errorf("open message = %+v err %v, want %v", f, err, want)
|
||||
}
|
||||
// Nothing open is a message too: the indexer then drops every candidate.
|
||||
m, err = OpenMessage([]byte(`{"pulls":[]}`))
|
||||
if f, derr := delivery.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 0 {
|
||||
if f, derr := forgemsg.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 0 {
|
||||
t.Errorf("empty answer: %+v %v %v", f, err, derr)
|
||||
}
|
||||
for _, bad := range []string{`{"pulls":null}`, `{}`, `x`} {
|
||||
@@ -366,24 +370,24 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6
|
||||
}
|
||||
|
||||
func TestTrackSaysWhichHeadOfWhichPullRequestWasWritten(t *testing.T) { // 1279/AC6
|
||||
pullMsg := func(action string) delivery.Forge {
|
||||
return delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action}
|
||||
pullMsg := func(action string) forgemsg.Forge {
|
||||
return forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action}
|
||||
}
|
||||
for _, a := range []string{delivery.PullOpened, delivery.PullUpdated} {
|
||||
for _, a := range []string{forgemsg.PullOpened, forgemsg.PullUpdated} {
|
||||
if key, h, ok := Track(pullMsg(a)); key != "novox.mesh-delivery.12" || h != head || !ok {
|
||||
t.Errorf("%s: Track = %q %q %v, want the head kept under a bucket key", a, key, h, ok)
|
||||
}
|
||||
}
|
||||
// A closed pull request and a merged one are no longer kept: their key goes.
|
||||
if key, h, ok := Track(pullMsg(delivery.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok {
|
||||
if key, h, ok := Track(pullMsg(forgemsg.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok {
|
||||
t.Errorf("closed: Track = %q %q %v", key, h, ok)
|
||||
}
|
||||
merged := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}
|
||||
merged := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}
|
||||
if key, h, ok := Track(merged); key != "novox.mesh-delivery.12" || h != "" || !ok {
|
||||
t.Errorf("merged: Track = %q %q %v", key, h, ok)
|
||||
}
|
||||
// The open message and the rest hold no head.
|
||||
if _, _, ok := Track(delivery.Forge{Kind: delivery.ForgeOpen}); ok {
|
||||
if _, _, ok := Track(forgemsg.Forge{Kind: forgemsg.ForgeOpen}); ok {
|
||||
t.Error("an open message was tracked")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// Caller calls a module's tool by its manifest key (`<module>.<tool>`) with a JSON body and returns the tool's result:
|
||||
// the bridge's process calls through the bus as its own principal, its tools bundle through the runtime (stdio.Ask).
|
||||
type Caller func(ctx context.Context, key string, body any) (json.RawMessage, error)
|
||||
|
||||
// CommitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner,
|
||||
// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4).
|
||||
const CommitTreesTool = "gitea.gitea_commit_trees"
|
||||
|
||||
// TreesVia asks the forge for a commit's trees through call.
|
||||
func TreesVia(ctx context.Context, call Caller) Trees {
|
||||
return func(owner, repo, sha string) (string, map[string]string, error) {
|
||||
raw, err := call(ctx, CommitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var a struct {
|
||||
Root string `json:"root"`
|
||||
Trees map[string]string `json:"trees"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", CommitTreesTool, err)
|
||||
}
|
||||
if a.Root == "" {
|
||||
return "", nil, fmt.Errorf("the answer of %s names no root tree", CommitTreesTool)
|
||||
}
|
||||
return a.Root, a.Trees, nil
|
||||
}
|
||||
}
|
||||
|
||||
// GetFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
|
||||
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
|
||||
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
|
||||
const GetFileTool = "gitea.gitea_get_file"
|
||||
|
||||
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
|
||||
// recognises it as /: 404 /).
|
||||
var notFound = regexp.MustCompile(`: 404 `)
|
||||
|
||||
// FilesVia asks the forge for a file through call. A file the forge says is not there is ErrNotFound.
|
||||
func FilesVia(ctx context.Context, call Caller) Files {
|
||||
return func(owner, repo, path, ref string) ([]byte, error) {
|
||||
raw, err := call(ctx, GetFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
|
||||
if err != nil {
|
||||
if notFound.MatchString(err.Error()) {
|
||||
return nil, fmt.Errorf("%w: %v", ErrNotFound, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", GetFileTool, err)
|
||||
}
|
||||
return []byte(*a.File.Content), nil
|
||||
}
|
||||
}
|
||||
|
||||
// MergedMessage is the merged message of a merge into the trunk, asked again: built as the bridge builds the one it writes
|
||||
// on FORGE (it is that event's body through ConvertKept), from the pull request's owner, repository, number, branch, head
|
||||
// and merge commit, which the gate's merges state holds. It reads the forge for every module directory (no kept inputs).
|
||||
func MergedMessage(owner, repo string, number int, branch, head, merge string, trees Trees, files Files) (Message, error) {
|
||||
body, err := json.Marshal(pull{Owner: owner, Repo: repo, Number: number, Head: branch, HeadSHA: head, Base: Trunk, MergeSHA: merge})
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
return ConvertKept(MergedSubject, body, trees, files, nil)
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the
|
||||
@@ -69,7 +69,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -90,7 +90,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil {
|
||||
if f, err = forgemsg.DecodeForge(msg.Data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.RetiredPlacements) > 0 || f.RetiredRead || strings.Contains(string(msg.Data), `"placements`) {
|
||||
@@ -108,7 +108,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
}
|
||||
forge.have[PredefinedFile] = []byte(`{"font-size":{"type":"integer","value":13}}`)
|
||||
msg, _ = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get)
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead {
|
||||
if f, err = forgemsg.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead {
|
||||
t.Errorf("predefined = %s (%v)", f.Predefined, err)
|
||||
}
|
||||
forge.calls = nil
|
||||
@@ -133,7 +133,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ := delivery.DecodeForge(msg.Data)
|
||||
f, _ := forgemsg.DecodeForge(msg.Data)
|
||||
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 1 || len(f.Packages) != 0 {
|
||||
t.Errorf("manifests %v locks %v packages %v", f.Manifests, f.Locks, f.Packages)
|
||||
}
|
||||
@@ -146,7 +146,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ = delivery.DecodeForge(msg.Data)
|
||||
f, _ = forgemsg.DecodeForge(msg.Data)
|
||||
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
|
||||
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
|
||||
}
|
||||
@@ -175,7 +175,7 @@ func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testin
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil {
|
||||
if f, _ := forgemsg.DecodeForge(msg.Data); f.Locks != nil {
|
||||
t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// ADR 0495 clause 1 with the lead's decision L8 of R4e: the forge bridge is no reader of a manifest's `paths`. A merge of a
|
||||
@@ -26,7 +26,7 @@ func TestAMergedMessageCarriesNoNamedPathsTheBridgeDerived(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
// Package groups is what the forge bridge's process and its tools bundle share of a delivery group's end (novox/hq ADR
|
||||
// 0633 §4, issue 1585): the keys of the bridge's state performed, and the asks of the state end-asks, through which the
|
||||
// tool bridge_group_end_again asks the process to say a group's end again (the bridge serves no request subject, ADR 0437
|
||||
// §4, so a tool asks in a state the process watches, as indexer_take_in asks the indexer in takeins).
|
||||
package groups
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The bridge's states the tools read and write.
|
||||
const (
|
||||
// PerformedState is the bridge's state of what it performed and published.
|
||||
PerformedState = "performed"
|
||||
// EndAsksState is the state of asks to say a group's end again, keyed by the gate's merges key of the group.
|
||||
EndAsksState = "end-asks"
|
||||
// EndAsksBucket is its bucket on the bus.
|
||||
EndAsksBucket = delivery.ForgeBridgeModule + "_" + EndAsksState
|
||||
)
|
||||
|
||||
// Key is the key of performed that says a group's end is written: `groups.<merges key>`, its value the revision of the
|
||||
// gate's merges key it was worked out from.
|
||||
func Key(mergesKey string) string { return "groups." + mergesKey }
|
||||
|
||||
// PublishedKey is the key of performed that says a member's merged message is written: `published.<16 hex digits of the
|
||||
// sha256 of <repository>#<number>@<merge>>`.
|
||||
func PublishedKey(repository string, number int, merge string) string {
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge)))
|
||||
return "published." + hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// EndID is the message id of a group's end, and of one said again by hand at a time: the stream's duplicate window drops a
|
||||
// second publish of the same id.
|
||||
func EndID(group, lastMerge, again string) string {
|
||||
id := "group-end." + group + "." + lastMerge
|
||||
if again != "" {
|
||||
id += "." + again
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// AgainStamp is the time a hand ask appends to a group end's message id.
|
||||
func AgainStamp(at time.Time) string { return at.UTC().Format("20060102T150405.000000000Z") }
|
||||
|
||||
// EndAsk is a value of the end-asks state: which group's end a person asked to be said again, why, who and when, and, once
|
||||
// the bridge read it, its answer.
|
||||
type EndAsk struct {
|
||||
V int `json:"v"`
|
||||
Group string `json:"group"`
|
||||
Why string `json:"why,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Written bool `json:"written,omitempty"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
}
|
||||
|
||||
// Done says the ask is answered.
|
||||
func (a EndAsk) Done() bool { return !a.Answered.IsZero() }
|
||||
|
||||
// NewEndAsk is the value bridge_group_end_again writes for a group: its key (the gate's merges key) and the value.
|
||||
func NewEndAsk(group, why, by string, at time.Time) (string, []byte, error) {
|
||||
group = strings.TrimSpace(group)
|
||||
if group == "" {
|
||||
return "", nil, errors.New("name the group whose end is said again (group)")
|
||||
}
|
||||
key, err := delivery.MergesKey(group)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
data, err := json.Marshal(EndAsk{V: 1, Group: group, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// DecodeEndAsk reads an end-asks value.
|
||||
func DecodeEndAsk(data []byte) (EndAsk, error) {
|
||||
var a EndAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if a.V != 1 || a.Group == "" {
|
||||
return a, errors.New("an end-asks value of no version 1 naming no group")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Row is one group as bridge_groups answers it: the gate's merge of it and what the bridge kept of its end.
|
||||
type Row struct {
|
||||
Key string `json:"key"`
|
||||
Group string `json:"group"`
|
||||
Members []RowMember `json:"members"`
|
||||
Ended bool `json:"ended"`
|
||||
EndedAt uint64 `json:"ended-at-revision,omitempty"`
|
||||
Open bool `json:"open"`
|
||||
Words string `json:"words"`
|
||||
// LastEndAsk is the last ask to say the group's end again (bridge_group_end_again), its who, why and when, and the
|
||||
// bridge's answer once it read it; absent when none was asked. The state end-asks keeps the last per group.
|
||||
LastEndAsk *EndAsk `json:"last-end-ask,omitempty"`
|
||||
}
|
||||
|
||||
// RowMember is one approved member of a group: its merge's state at the gate and whether its merged message is written.
|
||||
type RowMember struct {
|
||||
Repository string `json:"repository"`
|
||||
Number int `json:"number"`
|
||||
State string `json:"state"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Published bool `json:"published"`
|
||||
}
|
||||
|
||||
// Rows are the groups the gate's merges hold with more than one member, each with what the bridge kept: ended (its end
|
||||
// written, the revision it was worked out from) or open (some member is not merged or its merged message not written, or its
|
||||
// end not written). merges is the gate's merges by key, performed the bridge's performed keys by key; unread are the gate's
|
||||
// keys that do not read. asks are the end-asks values by the same key; one that does not read is named unread as
|
||||
// `end-asks.<key>`.
|
||||
func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[string]uint64, asks map[string][]byte) (rows []Row, unread []string) {
|
||||
lastAsk := map[string]*EndAsk{}
|
||||
for key, raw := range asks {
|
||||
a, err := DecodeEndAsk(raw)
|
||||
if err != nil {
|
||||
unread = append(unread, EndAsksState+"."+key)
|
||||
continue
|
||||
}
|
||||
lastAsk[key] = &a
|
||||
}
|
||||
for key, raw := range merges {
|
||||
m, err := delivery.DecodeMerge(raw)
|
||||
if err != nil {
|
||||
unread = append(unread, key)
|
||||
continue
|
||||
}
|
||||
if len(m.Approved) < 2 {
|
||||
continue
|
||||
}
|
||||
r := Row{Key: key, Group: m.Group}
|
||||
allMerged := true
|
||||
for _, p := range m.Pulls {
|
||||
pub := p.Commit != "" && performed[PublishedKey(p.Repository, p.Number, p.Commit)]
|
||||
r.Members = append(r.Members, RowMember{Repository: p.Repository, Number: p.Number, State: string(p.State), Commit: p.Commit, Published: pub})
|
||||
if p.State != delivery.MergeMerged || !pub {
|
||||
allMerged = false
|
||||
}
|
||||
}
|
||||
if len(m.Pulls) < len(m.Approved) {
|
||||
allMerged = false
|
||||
}
|
||||
r.EndedAt, r.Ended = endedAt[key]
|
||||
r.LastEndAsk = lastAsk[key]
|
||||
r.Open = !r.Ended
|
||||
switch {
|
||||
case r.Ended:
|
||||
r.Words = "its end is written"
|
||||
case allMerged:
|
||||
r.Words = "every member is merged and its message written, and its end is not written: bridge_group_end_again says it"
|
||||
default:
|
||||
r.Words = "gathered: not every member is merged and its merged message written"
|
||||
}
|
||||
rows = append(rows, r)
|
||||
}
|
||||
sortRows(rows)
|
||||
sort.Strings(unread)
|
||||
return rows, unread
|
||||
}
|
||||
|
||||
func sortRows(rows []Row) { sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) }
|
||||
|
||||
// MembersOf are a merge's approved members with their heads, in the order of their keys.
|
||||
func MembersOf(m delivery.Merge) []delivery.Head {
|
||||
var out []delivery.Head
|
||||
for _, k := range slices.Sorted(func(yield func(string) bool) {
|
||||
for k := range m.Approved {
|
||||
if !yield(k) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}) {
|
||||
at := strings.LastIndex(k, "@")
|
||||
if at < 0 {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(k[at+1:])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, delivery.Head{Repository: k[:at], Number: n, Head: m.Approved[k]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -28,12 +28,13 @@ type state struct {
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
@@ -44,6 +45,7 @@ type manifest struct {
|
||||
Streams []stream `json:"streams"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
@@ -107,11 +109,15 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
|
||||
// §5), the revision of each key of the gate's buckets the bridge performed, so a key is performed once per
|
||||
// revision (ADR 0437 §8; issue 1341), and what a merge read of each module directory by its tree, so a merge reads
|
||||
// the forge only for the directories it changed (issue 1126).
|
||||
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}}) { // 1279/AC6, 1341, 1126
|
||||
t.Errorf("state = %+v, want the buckets heads, performed and inputs", m.State)
|
||||
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}, {Name: "end-asks", History: 1}}) { // 1279/AC6, 1341, 1126, 1585
|
||||
t.Errorf("state = %+v, want the buckets heads, performed, inputs and end-asks", m.State)
|
||||
}
|
||||
// No other state, no bus, no seat, no tool, no event (ADR 0437 §4; ADR 0521: no check.status).
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
|
||||
// 1585/T18: the tools read the groups and ask the bridge to say a group's end again, through its state end-asks. The tool bridge_merged_message answers a merge's merged message again (issue 1585).
|
||||
if want := []string{"bridge_group_end_again", "bridge_groups", "bridge_merged_message"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// No other state, no bus, no seat, no event (ADR 0437 §4; ADR 0521: no check.status).
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the bridge has none", k)
|
||||
}
|
||||
@@ -143,8 +149,8 @@ func TestManifestProcess(t *testing.T) { // 1235/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
|
||||
@@ -13,7 +13,13 @@
|
||||
"state": [
|
||||
{ "name": "heads", "history": 1 },
|
||||
{ "name": "performed", "history": 1 },
|
||||
{ "name": "inputs", "history": 1 }
|
||||
{ "name": "inputs", "history": 1 },
|
||||
{ "name": "end-asks", "history": 1 }
|
||||
],
|
||||
"tools": [
|
||||
"bridge_groups",
|
||||
"bridge_group_end_again",
|
||||
"bridge_merged_message"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.merges",
|
||||
@@ -75,6 +81,17 @@
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-forge-bridge",
|
||||
"binary": "mesh-delivery-forge-bridge"
|
||||
},
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-forge-bridge-tools",
|
||||
"binary": "mesh-delivery-forge-bridge-tools",
|
||||
"loads": [
|
||||
"mesh-delivery-forge-bridge-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// catalogRead reads the indexer's modules, manifests and graph states for the seat's verbs.
|
||||
func catalogRead(modules, manifests, nodes stdio.KeptState) index.CatalogRead {
|
||||
return index.CatalogRead{
|
||||
Modules: func() (map[string]delivery.Module, error) {
|
||||
keys, err := modules.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]delivery.Module{}
|
||||
for _, k := range keys {
|
||||
e, err := modules.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("modules.%s: %w", k, err)
|
||||
}
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if m, err := delivery.DecodeModule(e.Value); err == nil {
|
||||
out[k] = m
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Manifest: func(digest string) (string, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
e, err := manifests.Get(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if e == nil {
|
||||
return "", fmt.Errorf("the manifests state holds no %s", digest)
|
||||
}
|
||||
kept, err := delivery.DecodeManifest(e.Value)
|
||||
return kept.JSON, err
|
||||
},
|
||||
Graph: func() (graph.Graph, error) {
|
||||
keys, err := nodes.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
g := graph.Graph{}
|
||||
for _, k := range keys {
|
||||
e, err := nodes.Get(k)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("graph.%s: %w", k, err)
|
||||
}
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if n, err := graph.Decode(e.Value); err == nil {
|
||||
g[k] = n
|
||||
}
|
||||
}
|
||||
return g, nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// catalogTools are the seat's verbs over the indexer's own states, served beside the indexer's own tools, since the indexer's manifest claims the seat (ADR 0604 §2).
|
||||
func catalogTools() []stdio.Tool {
|
||||
return index.CatalogTools(catalogRead(stdio.State("modules"), stdio.State("manifests"), stdio.State(graph.State)))
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
// Command mesh-delivery-indexer-tools serves the indexer's doing tool (novox/hq issue 1945): the runtime launches it for a
|
||||
// call. indexer_take_in writes one value of the indexer's takeins state and waits for the indexer's answer there. The
|
||||
// indexer is the other program: it serves no request subject (ADR 0437 §4). The read tools of the indexer's states are
|
||||
// indexer is the other program: it serves no request subject (ADR 0437 §4); indexer_judge asks the same way, and
|
||||
// indexer_verdicts reads the indexer's verdicts state. The read tools of the indexer's states are
|
||||
// mesh-delivery-view's (indexer_module, indexer_modules, indexer_candidate, indexer_candidates).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -21,10 +23,23 @@ import (
|
||||
// asker writes a takeins value and waits for the indexer's answer to it.
|
||||
type asker func(key string, value []byte) (index.TakeInAsk, error)
|
||||
|
||||
// Tools are the indexer's tools.
|
||||
func Tools(ask asker, now func() time.Time) []stdio.Tool {
|
||||
// Tools are the indexer's tools. read is the indexer's state of verdicts, which indexer_verdicts reads; counts the old sections' count.
|
||||
func Tools(ask asker, now func() time.Time, read stdio.KeptState, counts func() (index.OldSectionsCount, error)) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: index.OldSectionsTool,
|
||||
Description: "How many accepted modules use each old section of a manifest (state, logs, streams, objects), the foundation " +
|
||||
"(nats, mesh-grants, the indexer) left out: the count the judge refuses a use of a section at 0 by, as modules move " +
|
||||
"their bus state into `requires` by template. Read from the indexer's oldsections state, kept as the indexer judges " +
|
||||
"the manifests; an empty answer means the indexer has not judged since it started.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
c, err := counts()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c.Counts, nil
|
||||
}},
|
||||
{Name: index.TakeInTool,
|
||||
Description: "Take a registered module in now: the indexer reads its manifest from the repository and commit the controller's " +
|
||||
"build records register it from (at the head the gate's source names, when it names one), whatever it tried before, and " +
|
||||
@@ -34,12 +49,70 @@ func Tools(ask asker, now func() time.Time) []stdio.Tool {
|
||||
"indexer) is read again at that head and written when it changed. Answers stored, from which " +
|
||||
"repository, commit and path, or why not (held already, no registration, the forge's error, no manifest there). " +
|
||||
"The indexer reads such a module again by itself on the next source revision and merged message; this asks now. " +
|
||||
"indexer_module (mesh-delivery-view) reads the result.",
|
||||
"indexer_module (mesh-delivery-view) reads the result. With repository instead of module it takes in the head the gate's " +
|
||||
"source names for that repository, for a merge whose merged message the indexer never heard (the forge bridge has no " +
|
||||
"tool): a head whose trunk holds no module.json is taken in as a merge that leaves none (every module of the repository " +
|
||||
"is dropped, one another repository's trunk holds handed over from there first, listed in handed and dropped); a head " +
|
||||
"that holds modules is answered \"take each module in by name\".",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||
"repository": map[string]any{"type": "string", "description": "instead of module: the repository (owner/name) whose head is taken in"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
var key string
|
||||
var value []byte
|
||||
var err error
|
||||
if str(args, "repository") != "" && str(args, "module") == "" {
|
||||
key, value, err = index.NewTakeInRepository(str(args, "repository"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
} else {
|
||||
key, value, err = index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: index.VerdictsTool,
|
||||
Description: "The indexer's verdicts on manifests that wait or are refused (all: every verdict), each as its verdicts state holds it: " +
|
||||
"the module, the digest, the state (waits: the manifest carries a field the indexer's judge does not know yet, named in " +
|
||||
"unknown, and every reader holds the module as a refused one and keeps what runs; refused: the judge's problems), the " +
|
||||
"judge that made it and when. A module's name narrows it to that module. A waiting manifest is judged again at each " +
|
||||
"new indexer and refused only once the newest build of the indexer on the trunk still does not know the field; " +
|
||||
"indexer_judge judges now.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "only this module's verdicts"},
|
||||
"all": map[string]any{"type": "boolean", "description": "list accepted verdicts too"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
keys, err := read.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the verdicts state cannot be listed: %w", err)
|
||||
}
|
||||
all, _ := args["all"].(bool)
|
||||
rows, unread := index.ListVerdicts(keys, func(key string) ([]byte, error) {
|
||||
e, err := read.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.Value, nil
|
||||
}, str(args, "module"), all)
|
||||
return map[string]any{"verdicts": rows, "unread": unread}, nil
|
||||
}},
|
||||
{Name: index.JudgeAgainTool,
|
||||
Description: "Judge the manifests again now, as the indexer does at its start and after each merge, and answer how the module's " +
|
||||
"verdict stands (waits, refused or accepted, and why). For a manifest that waits for a judge that knows its fields, or " +
|
||||
"after a manifest was mended where the indexer has not heard it. A waiting manifest becomes refused here only when the " +
|
||||
"running indexer is the newest build of itself the trunk holds. With a digest it judges that one manifest of the " +
|
||||
"module (a candidate's, or one an older judge refused) in the catalogue the trunk holds. indexer_verdicts reads the result.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name, as the catalogue registers it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in now, in plain words"}}},
|
||||
"digest": map[string]any{"type": "string", "description": "one manifest digest of the module to judge (indexer_verdicts names it); none judges every manifest the trunk holds"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is judged again now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := index.NewTakeIn(str(args, "module"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
key, value, err := index.NewJudgeAgain(str(args, "module"), str(args, "digest"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -105,8 +178,47 @@ func askTakeIn(s stdio.KeptState, bound time.Duration) asker {
|
||||
}
|
||||
}
|
||||
|
||||
// heldCounts reads the count of the old sections the indexer keeps.
|
||||
func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) {
|
||||
return func() (index.OldSectionsCount, error) {
|
||||
e, err := s.Get(index.OldSectionsKey)
|
||||
if err != nil {
|
||||
return index.OldSectionsCount{}, fmt.Errorf("reading %s.%s: %w", index.OldSectionsBucket, index.OldSectionsKey, err)
|
||||
}
|
||||
if e == nil {
|
||||
return index.OldSectionsCount{}, fmt.Errorf("the indexer has not counted the old sections yet: %s holds no %s", index.OldSectionsBucket, index.OldSectionsKey)
|
||||
}
|
||||
return index.DecodeOldSections(e.Value)
|
||||
}
|
||||
}
|
||||
|
||||
// keptRead is a state the tools bundle reads, as the index package reads a store: Get and Keys; it writes nothing.
|
||||
type keptRead struct{ s stdio.KeptState }
|
||||
|
||||
func (k keptRead) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := k.s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (k keptRead) Keys(context.Context) ([]string, error) { return k.s.Keys() }
|
||||
|
||||
func (keptRead) Create(context.Context, string, []byte) error { return errReadOnly }
|
||||
|
||||
func (keptRead) Update(context.Context, string, []byte, uint64) error { return errReadOnly }
|
||||
|
||||
func (keptRead) Delete(context.Context, string, uint64) error { return errReadOnly }
|
||||
|
||||
var errReadOnly = errors.New("the tools bundle only reads this state")
|
||||
|
||||
func main() {
|
||||
if err := stdio.Serve("", Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now)); err != nil {
|
||||
ask := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second)
|
||||
tools := append(Tools(ask, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...)
|
||||
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, ask, time.Now)...)
|
||||
tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, ask, time.Now)...)
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// mergedTool is the forge bridge's tool that answers a merge's merged message again (issue 1585): arguments repository
|
||||
// and merge, answer {message}, the message whole as the bridge writes it on FORGE. The indexer's manifest lists it in
|
||||
// invokes, which is the grant to call it.
|
||||
const mergedTool = delivery.ForgeBridgeModule + ".bridge_merged_message"
|
||||
|
||||
// messageVia asks the forge bridge for a merge's merged message, for a held merge the indexer does not hold in memory (a
|
||||
// restart, a new build). An error is the bridge's own words.
|
||||
func messageVia(call toolCaller) func(ctx context.Context, repository, merge string) ([]byte, error) {
|
||||
return func(_ context.Context, repository, merge string) ([]byte, error) {
|
||||
raw, err := call(mergedTool, map[string]string{"repository": repository, "merge": merge})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Message json.RawMessage `json:"message"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || len(a.Message) == 0 {
|
||||
return nil, fmt.Errorf("the answer of %s is not {message}: %v", mergedTool, err)
|
||||
}
|
||||
return a.Message, nil
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,10 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -17,9 +20,15 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
@@ -42,22 +51,89 @@ accounts {
|
||||
// bus with the stream FORGE and the indexer's durable <node>_<module> on it, the buckets the controller makes from the
|
||||
// manifest's state (mesh-delivery-indexer_candidates, history 10) and mesh-controller_builds holding build records, and
|
||||
// no mesh-delivery-gate_source bucket.
|
||||
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
|
||||
js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) {
|
||||
//
|
||||
// And the take-in step (ADR 0633, issue 1585 I8a, I8b): the indexer's states pending, verdicts, graph, taken and packages
|
||||
// and its stream take-in are there, mesh-build-go reported its toolchain, and the node-engine's composition answers the
|
||||
// build of the indexer that runs. A merge changing the indexer's own tree writes only the indexer's entry, its manifest,
|
||||
// its verdict and its key of the graph (and the packages it publishes); the rest is pending, said take-in.pending and
|
||||
// named by indexer_pending with the build it waits for. A group's first member is gathered meanwhile. Started again as
|
||||
// that build, the indexer takes the revision in and leaves the group gathered; the group's second member and its end
|
||||
// then make one take-in, said take-in.taken under the group's key.
|
||||
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
var running atomic.Pointer[string]
|
||||
first := "fp-the-indexer-that-runs"
|
||||
running.Store(&first)
|
||||
// The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by
|
||||
// repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it.
|
||||
var bridgeMessages sync.Map
|
||||
js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
|
||||
// The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them.
|
||||
builds.end(t, "novox/mesh-delivery", "a", "modules/a")
|
||||
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
|
||||
})
|
||||
h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef"
|
||||
for _, f := range []delivery.Forge{
|
||||
{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true},
|
||||
{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true},
|
||||
} {
|
||||
data, err := delivery.EncodeForge(f)
|
||||
}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
// The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11).
|
||||
for bucket, history := range map[string]uint8{index.PendingBucket: 1, delivery.VerdictsBucket: delivery.VerdictsHistory,
|
||||
graph.Bucket: 1, delivery.TakenBucket: 1, delivery.PackagesBucket: 1} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The take-in events are the indexer's ordinary events (ADR 0633 §5), on the path manifest-accepted takes.
|
||||
for name, subject := range map[string]string{"INDEXER_EVENTS": "mesh.mod.mesh-delivery-indexer.event.>"} {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: name, Subjects: []string{subject}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event, _ := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
key, _ := delivery.ToolchainKey("n1", "go")
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tc.Put(ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there.
|
||||
answer := func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"})
|
||||
_ = m.Respond(raw)
|
||||
}
|
||||
bridge := func(m *nats.Msg) {
|
||||
var ask struct{ Repository, Merge string }
|
||||
_ = json.Unmarshal(m.Data, &ask)
|
||||
data, ok := bridgeMessages.Load(ask.Repository + "@" + ask.Merge)
|
||||
if !ok {
|
||||
raw, _ := json.Marshal(map[string]any{"error": "the gate holds no merge of " + ask.Repository + " with that commit"})
|
||||
_ = m.Respond(raw)
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"message": json.RawMessage(data.([]byte))}, "node": "n1"})
|
||||
_ = m.Respond(raw)
|
||||
}
|
||||
for _, subject := range []string{"mesh.mod.mesh-delivery-forge-bridge.call.bridge_merged_message.>", "mesh.mod.mesh-delivery-forge-bridge.tool.bridge_merged_message"} {
|
||||
if _, err := js.Conn().Subscribe(subject, bridge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, subject := range []string{"mesh.seat.node-engine.call.composition.n1.>", "mesh.seat.node-engine.tool.composition.n1"} {
|
||||
if _, err := js.Conn().Subscribe(subject, answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef"
|
||||
for _, f := range []forgemsg.Forge{
|
||||
{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true},
|
||||
{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true},
|
||||
} {
|
||||
data, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -83,11 +159,202 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
|
||||
t.Errorf("%s#%d modules = %v, want [%s]", p.Repository, p.Number, p.Modules, want)
|
||||
}
|
||||
}
|
||||
|
||||
// I8a: pull request 12 merged, changing the indexer's own tree and a's.
|
||||
manifest, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const ix = delivery.IndexerModule
|
||||
m1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1,
|
||||
Merge: "1111111111111111111111111111111111111111", Root: "r0",
|
||||
Trees: map[string]string{"modules/" + ix: "tree-indexer-2", "modules/a": "tree-a-2"},
|
||||
Manifests: map[string]string{"modules/" + ix: string(manifest), "modules/a": `{"module":"a","version":"1"}`},
|
||||
Locks: map[string]map[string]string{"modules/" + ix: {"go.sum": "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"}}}
|
||||
merged := func(f forgemsg.Forge) uint64 {
|
||||
t.Helper()
|
||||
data, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgeMerged)
|
||||
bridgeMessages.Store(f.Repository+"@"+f.Merge, data)
|
||||
ack, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ack.Sequence
|
||||
}
|
||||
m1Key := takein.PendingKey(merged(m1))
|
||||
events, err := js.Stream(ctx, "INDEXER_EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// saidAll is every take-in event of kind said under key so far, oldest first.
|
||||
saidAll := func(kind, key string) []takein.TakeIn {
|
||||
event, _ := takein.TakeInEvent(kind)
|
||||
info, err := events.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []takein.TakeIn
|
||||
for seq := info.State.FirstSeq; seq <= info.State.LastSeq && seq > 0; seq++ {
|
||||
msg, err := events.GetMsg(ctx, seq)
|
||||
if err != nil || msg.Subject != "mesh.mod."+ix+".event."+event {
|
||||
continue
|
||||
}
|
||||
if ti, err := takein.DecodeTakeIn(msg.Data); err == nil && ti.Key == key {
|
||||
out = append(out, ti)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// said is the last take-in event under kind and key, within 20 s.
|
||||
said := func(kind, key string) (takein.TakeIn, bool) {
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if all := saidAll(kind, key); len(all) > 0 {
|
||||
return all[len(all)-1], true
|
||||
}
|
||||
}
|
||||
return takein.TakeIn{}, false
|
||||
}
|
||||
held, ok := said(takein.TakeInPending, m1Key)
|
||||
if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 {
|
||||
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first)
|
||||
}
|
||||
named := held.WaitsFor
|
||||
get := func(bucket, key string) ([]byte, bool) {
|
||||
t.Helper()
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return e.Value(), true
|
||||
}
|
||||
// Only what building and moving the named indexer needs is written.
|
||||
raw, ok := get(delivery.ModulesBucket, ix)
|
||||
own, err := delivery.DecodeModule(raw)
|
||||
if !ok || err != nil || own.FP2 != named || own.Commit != m1.Merge {
|
||||
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
|
||||
}
|
||||
if _, ok := get(delivery.ManifestsBucket, own.Manifest); !ok {
|
||||
t.Errorf("manifests.%s is not written", own.Manifest)
|
||||
}
|
||||
if _, ok := get(delivery.VerdictsBucket, own.Manifest); !ok {
|
||||
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
|
||||
}
|
||||
if gk, _ := graph.Key(ix); !func() bool { _, ok := get(graph.Bucket, gk); return ok }() {
|
||||
t.Errorf("the indexer's key %s of the module graph is not written", gk)
|
||||
}
|
||||
repoKey, _ := delivery.RepositoryKey(m1.Repository)
|
||||
if _, ok := get(delivery.ModulesBucket, "a"); ok {
|
||||
t.Errorf("modules.a is written while the revision is pending")
|
||||
}
|
||||
if _, ok := get(delivery.TakenBucket, repoKey); ok {
|
||||
t.Errorf("taken.%s is written while the revision is pending", repoKey)
|
||||
}
|
||||
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
|
||||
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
|
||||
} else if c, err := delivery.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
|
||||
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
|
||||
}
|
||||
|
||||
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
|
||||
members := []delivery.Head{{Repository: "novox/mesh-catalog", Number: 40, Head: "4040404040404040404040404040404040404040"},
|
||||
{Repository: "novox/mesh-controller", Number: 41, Head: "4141414141414141414141414141414141414141"}}
|
||||
g1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/g", Number: 40, Head: members[0].Head,
|
||||
Merge: "4000000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/c": "tree-c-2"},
|
||||
Manifests: map[string]string{"modules/c": `{"module":"c","version":"1"}`}, Group: "feat/g", Members: members}
|
||||
g2 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-controller", Branch: "feat/g", Number: 41, Head: members[1].Head,
|
||||
Merge: "4100000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/k": "tree-k-1"},
|
||||
Manifests: map[string]string{"modules/k": `{"module":"k","version":"1"}`}, Group: "feat/g", Members: members}
|
||||
g1Key, groupKey := takein.PendingKey(merged(g1)), takein.GroupTakeInKey("feat/g")
|
||||
if gathered, ok := said(takein.TakeInPending, groupKey); !ok || gathered.What != takein.TakeInGroup || len(gathered.Merges) != 1 {
|
||||
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the group with its first member", groupKey, gathered, ok)
|
||||
}
|
||||
// indexer_pending, the tool over the same state, names both waits.
|
||||
pendingKV, err := js.KeyValue(ctx, index.PendingBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waits := func() []takein.PendingWait {
|
||||
t.Helper()
|
||||
answer, err := index.PendingTools(kvStore{pendingKV}, nil, nil, time.Now)[0].Run(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return answer.(takein.PendingAnswer).Waits
|
||||
}
|
||||
var w []takein.PendingWait
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) ||
|
||||
w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent {
|
||||
t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named)
|
||||
}
|
||||
|
||||
// The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group.
|
||||
running.Store(&named)
|
||||
restart()
|
||||
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
|
||||
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named)
|
||||
}
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if w = waits(); len(w) == 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw, ok := get(delivery.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
|
||||
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
|
||||
}
|
||||
if _, ok := get(delivery.TakenBucket, repoKey); !ok {
|
||||
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
|
||||
}
|
||||
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
|
||||
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
|
||||
}
|
||||
if _, ok := get(delivery.ModulesBucket, "c"); ok {
|
||||
t.Errorf("modules.c is written while its group is gathered")
|
||||
}
|
||||
|
||||
// I8b: the group's second member and its end: one take-in of both, said under the group's key.
|
||||
merged(g2)
|
||||
end, err := forgemsg.EncodeForgeGroupEnd(forgemsg.ForgeGroupEnd{Group: "feat/g", Members: []forgemsg.GroupMerge{
|
||||
{Repository: g1.Repository, Number: g1.Number, Merge: g1.Merge}, {Repository: g2.Repository, Number: g2.Number, Merge: g2.Merge}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+forgemsg.ForgeGroupEndEvent, end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if taken, ok := said(takein.TakeInTaken, groupKey); !ok || taken.What != takein.TakeInGroup || len(taken.Merges) != 2 {
|
||||
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the group's end, want both members", groupKey, taken, ok)
|
||||
}
|
||||
var hasC, hasK bool
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) {
|
||||
_, hasC = get(delivery.ModulesBucket, "c")
|
||||
_, hasK = get(delivery.ModulesBucket, "k")
|
||||
w = waits()
|
||||
}
|
||||
if !hasC || !hasK || len(w) != 0 {
|
||||
t.Errorf("after the group's end: modules.c %v, modules.k %v, indexer_pending %+v: want both taken in and nothing waiting", hasC, hasK, w)
|
||||
}
|
||||
// One take-in: neither member was said as a revision of its own.
|
||||
if len(saidAll(takein.TakeInPending, g1Key)) != 0 {
|
||||
t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key)
|
||||
}
|
||||
}
|
||||
|
||||
// startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
|
||||
// mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created.
|
||||
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
|
||||
// startIndexerRun starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
|
||||
// mesh-controller_builds, and runs the program; restart starts it again. mesh-delivery-gate_source is never created.
|
||||
func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue, func()) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -166,11 +433,24 @@ func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...f
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
var stop func()
|
||||
start := func() {
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
stop = func() { cancel(); <-done }
|
||||
}
|
||||
start()
|
||||
t.Cleanup(func() { stop() })
|
||||
// restart stops the program and starts it again on the same bus, as the node-engine does with a new build.
|
||||
restart := func() { stop(); start() }
|
||||
|
||||
return js, ctx, candidates, modules, restart
|
||||
}
|
||||
|
||||
// startIndexer is startIndexerRun for a test that never starts the program again.
|
||||
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
|
||||
js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...)
|
||||
return js, ctx, candidates, modules
|
||||
}
|
||||
|
||||
@@ -202,7 +482,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
|
||||
builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent")
|
||||
})
|
||||
const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"},
|
||||
Manifests: map[string]string{"modules/a": `{"module":"a","version":"1"}`}})
|
||||
if err != nil {
|
||||
@@ -229,7 +509,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
|
||||
// its inputs and the reason it has no fingerprint of the new form, and manifests.<digest> with the manifest byte for byte.
|
||||
//
|
||||
// And it judges them (novox/hq ADR 0578, issue 1585): with three manifests more, one valid, one refused and one with a
|
||||
// field the decoder does not know, mesh-sdk's bus reader read before the indexer judged composes all three, the field set
|
||||
// field the decoder does not know (it waits, ADR 0601 §3), mesh-sdk's bus reader read before the indexer judged composes all three, the field set
|
||||
// aside and named (a missing verdict is accepted, §3); the indexer writes the three verdicts, says manifest-refused for
|
||||
// each refusal, and the reader then composes the valid one and holds the others, named, never ending its read.
|
||||
func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 1585
|
||||
@@ -256,7 +536,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
forge := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
forge := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
@@ -264,7 +544,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
|
||||
for name, raw := range judged {
|
||||
forge.Manifests["modules/"+name], forge.Trees["modules/"+name] = raw, "tree-"+name
|
||||
}
|
||||
data, err := delivery.EncodeForge(forge)
|
||||
data, err := forgemsg.EncodeForge(forge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -300,7 +580,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"good": delivery.VerdictAccepted, "bad": delivery.VerdictRefused, "later": delivery.VerdictRefused}
|
||||
want := map[string]string{"good": delivery.VerdictAccepted, "bad": delivery.VerdictRefused, "later": delivery.VerdictManifestWaits}
|
||||
got := map[string]delivery.ManifestVerdict{}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && len(got) < len(want); time.Sleep(50 * time.Millisecond) {
|
||||
for name := range want {
|
||||
@@ -316,10 +596,12 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
|
||||
t.Errorf("the verdict on %s = %+v within 10 s, want %s by the running judge", name, v, state)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(got["bad"].Problems, " "), `provides "amqp"`) || !strings.Contains(strings.Join(got["later"].Problems, " "), "a-field-from-later") {
|
||||
t.Errorf("the refusals name no problem: %+v", got)
|
||||
// ADR 0601 §3: the field this judge does not know waits and is named, since no node-engine answers which build of the
|
||||
// indexer runs here (the unit tests of the index package prove it refused once that build is the trunk's).
|
||||
if !strings.Contains(strings.Join(got["bad"].Problems, " "), `provides "amqp"`) || strings.Join(got["later"].Unknown, " ") != "a-field-from-later" {
|
||||
t.Errorf("the refusal names no problem or the wait no field: %+v", got)
|
||||
}
|
||||
// manifest-refused, said once for each refusal.
|
||||
// manifest-refused, said once for each refusal (a wait is no refusal).
|
||||
events, err := js.Stream(ctx, "INDEXER_EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -335,8 +617,8 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
|
||||
said[r.Module] = true
|
||||
}
|
||||
}
|
||||
if !said["bad"] || !said["later"] || said["good"] {
|
||||
t.Errorf("manifest-refused said for %v, want bad and later", said)
|
||||
if !said["bad"] || said["later"] || said["good"] {
|
||||
t.Errorf("manifest-refused said for %v, want bad only", said)
|
||||
}
|
||||
// How a composer holds a refused module by these verdicts is the node-engine's to test (ADR 0593 §7: the composer
|
||||
// and its bus reader are its own).
|
||||
@@ -356,7 +638,7 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes
|
||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) {
|
||||
builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi")
|
||||
})
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
@@ -443,7 +725,7 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
pull, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: delivery.PullUpdated,
|
||||
pull, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: forgemsg.PullUpdated,
|
||||
Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": string(manifest)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -613,7 +895,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
place(ctx, t, "n1", "mesh-build-go")
|
||||
place(ctx, t, "n2")
|
||||
})
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
|
||||
@@ -24,8 +24,11 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -151,6 +154,34 @@ type delivered interface {
|
||||
Metadata() (*jetstream.MsgMetadata, error)
|
||||
}
|
||||
|
||||
// groupEndSubject is the whole subject of a delivery group's end on the forge bridge's stream: compared exactly, so a
|
||||
// repository named forge/group, whose merged message ends the same way, is never taken for one.
|
||||
var groupEndSubject = process.ModuleStreamSubject(delivery.ForgeBridgeModule, forgemsg.ForgeGroupEndEvent)
|
||||
|
||||
// handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end
|
||||
// whose members are not all held yet, or while the build this program runs is not known, is left for redelivery.
|
||||
func handleGroupEnd(ctx context.Context, m delivered, ix *index.Indexer, h *index.Heard, logger *log.Logger) {
|
||||
end, err := forgemsg.DecodeForgeGroupEnd(m.Data())
|
||||
if err != nil {
|
||||
logger.Printf("%s: not a group's end, dropped: %v", m.Subject(), err)
|
||||
h.Outcome, h.Words = index.HeardDropped, err.Error()
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
h.Kind, h.Branch = forgemsg.ForgeGroupEndEvent, end.Group
|
||||
applied, words, err := ix.EndGroup(ctx, end)
|
||||
h.Written, h.Late = applied.Written, applied.Late
|
||||
if err != nil {
|
||||
logger.Printf("%s: the end of the group %q left for redelivery: %v", m.Subject(), end.Group, err)
|
||||
h.Outcome, h.Words = index.HeardLeft, err.Error()
|
||||
return
|
||||
}
|
||||
h.Outcome, h.Words = index.HeardApplied, words
|
||||
if err := m.DoubleAck(ctx); err != nil {
|
||||
logger.Printf("%s: the end of the group %q applied, its acknowledgement was not confirmed: %v", m.Subject(), end.Group, err)
|
||||
}
|
||||
}
|
||||
|
||||
// handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated,
|
||||
// never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock). Each message handled is written to the heard state
|
||||
@@ -169,7 +200,12 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto
|
||||
logger.Printf("%s: sequence %d is not written to the heard state: %v", h.Subject, h.Sequence, err)
|
||||
}
|
||||
}()
|
||||
f, err := delivery.DecodeForge(m.Data())
|
||||
// A delivery group's end (novox/hq ADR 0633 §4) has a body of its own: told apart by its name, before the decode.
|
||||
if m.Subject() == groupEndSubject {
|
||||
handleGroupEnd(ctx, m, ix, &h, logger)
|
||||
return
|
||||
}
|
||||
f, err := forgemsg.DecodeForge(m.Data())
|
||||
if err != nil {
|
||||
logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err)
|
||||
h.Outcome, h.Words = index.HeardDropped, err.Error()
|
||||
@@ -180,7 +216,13 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto
|
||||
if f.Merge != "" {
|
||||
h.Commit = f.Merge
|
||||
}
|
||||
applied, err := ix.ApplyMessage(ctx, f)
|
||||
var applied index.Applied
|
||||
if f.Kind == forgemsg.ForgeMerged && h.Sequence > 0 {
|
||||
// A merge is taken in only by the indexer its revision names (ADR 0633 §1): kept in pending until then.
|
||||
applied, h.Words, err = ix.Arrive(ctx, h.Sequence, f)
|
||||
} else {
|
||||
applied, err = ix.ApplyMessage(ctx, f)
|
||||
}
|
||||
h.Written, h.Late = applied.Written, applied.Late
|
||||
switch {
|
||||
case errors.Is(err, index.ErrUnreadable):
|
||||
@@ -296,11 +338,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The published packages (ADR 0593 §2): a state this delivery brings, bound when it appears; until then every
|
||||
// fingerprint takes the version line.
|
||||
packages := process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "no package is published, so every fingerprint takes the version line", logger.Printf)
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
|
||||
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf),
|
||||
// How many modules use each old section (issue 1966 D4): a state this delivery brings, bound when it appears; until then
|
||||
// the judge refuses no use of an old section.
|
||||
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
|
||||
// The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written.
|
||||
graph := process.AwaitBucket(ctx, js, conn, graph.Bucket, "no module graph is written", logger.Printf)
|
||||
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
|
||||
tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf)
|
||||
ix := &index.Indexer{OwnBuildState: awaitedStore{ownBuildState}, TryRecovery: awaitedStore{tryRecovery}, OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
|
||||
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
|
||||
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call)}
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), ReadFile: readFileVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), Graph: awaitedStore{graph}, Message: messageVia(calls.Call),
|
||||
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
|
||||
// until then every merge is taken in at once, as before.
|
||||
Pending: awaitedStore{process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
|
||||
SayTakeIn: sayTakeIn(js, cred)}
|
||||
// Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the
|
||||
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
|
||||
// revisions in with its old judge.
|
||||
ix.FixOwnBuild(ctx)
|
||||
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
|
||||
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
|
||||
var started atomic.Bool
|
||||
@@ -360,7 +418,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if !gate.Toolchain(module, key, value, deleted) {
|
||||
return nil
|
||||
}
|
||||
return ix.Recompute(ctx)
|
||||
// The modules written again, and pending evaluated again (ADR 0633 §1): the build a revision names moves with
|
||||
// the toolchain.
|
||||
return ix.ToolchainReported(ctx)
|
||||
},
|
||||
recompute: ix.Recompute}, logger)
|
||||
if err := tools.start(); err != nil {
|
||||
@@ -421,8 +481,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
logger.Printf("judging the manifests at start: %v; judged again at the next message heard", err)
|
||||
return
|
||||
}
|
||||
logger.Printf("judged %d manifests at start: %d verdicts written, %d with problems, of which outgrown and kept accepted: %v",
|
||||
j.Judged, j.Written, len(j.Refused), j.Outgrown)
|
||||
logger.Printf("judged %d manifests at start: %d verdicts written, %d with problems, of which outgrown and kept accepted: %v, waiting for a judge that knows their fields: %v",
|
||||
j.Judged, j.Written, len(j.Refused), j.Outgrown, j.Waiting)
|
||||
})
|
||||
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
|
||||
// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
|
||||
@@ -440,11 +500,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
defer tc.Stop()
|
||||
}
|
||||
switch pcons, err := cred.EventConsumer(ctx, js, PreviewsStream); {
|
||||
case err != nil:
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err)
|
||||
default:
|
||||
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.Written(m) }, logger.Printf)
|
||||
// The consumer on the checker's stream exists before the open tries are listed and is heard after (ADR 0638 §3): an event
|
||||
// said between the call and the hearing waits in it, and a try that started or ended meanwhile is found either way.
|
||||
pcons, perr := cred.EventConsumer(ctx, js, PreviewsStream)
|
||||
if perr != nil {
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews and tries are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, perr)
|
||||
}
|
||||
// The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's
|
||||
// stream starts (ADR 0633 §2). When the build this program runs is not known yet, nothing waits for it: the consumer
|
||||
// starts, and a merged message left unacknowledged asks the build again at its redelivery (clause 1). A hand take-in
|
||||
// (indexer_take_in_pending) is answered at any time: the takeins watch runs already.
|
||||
if _, words, err := ix.TakeInAtStart(ctx); err != nil {
|
||||
return fmt.Errorf("the take-in at start: %w", err)
|
||||
} else if words != "" {
|
||||
logger.Printf("take-in at start: %s", words)
|
||||
}
|
||||
// The open tries are found again through the checker's tool: a try event missed while the indexer was down would leave a
|
||||
// try with no candidate, or a candidate with no try. Not fatal: the events come, and a preview written writes the candidates.
|
||||
// A recovery that fails is said (journal, and indexer_try_candidates) and the events are heard all the same.
|
||||
_ = ix.RecoverTries(ctx)
|
||||
if perr == nil {
|
||||
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -73,8 +74,8 @@ const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
ix := &index.Indexer{Store: st, Builds: index.NewBuilds()}
|
||||
data, _ := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
|
||||
data, _ := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
|
||||
heard := &memStore{vals: map[string][]byte{}}
|
||||
handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0))
|
||||
@@ -84,7 +85,7 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
}
|
||||
// Issue 1126 note 76: what the indexer heard of FORGE is readable (indexer_heard): subject, sequence, deliveries, time,
|
||||
// repository, commit and what it did.
|
||||
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != delivery.ForgePull ||
|
||||
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != forgemsg.ForgePull ||
|
||||
h.Repository != "novox/mesh-delivery" || h.Number != 12 || h.Commit != head || h.Outcome != index.HeardApplied || h.Handled.IsZero() {
|
||||
t.Errorf("heard = %+v", h)
|
||||
}
|
||||
@@ -102,6 +103,31 @@ func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 12
|
||||
if h := heardOf(t, heard); h.Outcome != index.HeardDropped || h.Sequence != 41 || h.Words == "" {
|
||||
t.Errorf("heard = %+v", h)
|
||||
}
|
||||
// 1585/T9 (ADR 0633 §4): a group's end is told apart by its name before the forge decode: one that reads is acked
|
||||
// (an indexer with no pending state takes the members in one by one), one that does not is dropped as a group's end.
|
||||
for _, c := range []struct {
|
||||
data string
|
||||
termd int
|
||||
logged string
|
||||
}{{`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`, 0, ""},
|
||||
{`{"v":1,"group":"feat/x"}`, 1, "not a group's end"}} {
|
||||
logs.Reset()
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event." + forgemsg.ForgeGroupEndEvent, data: []byte(c.data)}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
|
||||
if m.termd != c.termd || m.acked+m.termd != 1 || !strings.Contains(logs.String(), c.logged) {
|
||||
t.Errorf("group end %s: terminated %d acked %d log %q", c.data, m.termd, m.acked, logs.String())
|
||||
}
|
||||
if h := heardOf(t, heard); h.Kind != forgemsg.ForgeGroupEndEvent && c.termd == 0 {
|
||||
t.Errorf("heard of a group's end = %+v", h)
|
||||
}
|
||||
}
|
||||
// The repository forge/group's merged message ends as a group's end does: it is a forge message all the same.
|
||||
logs.Reset()
|
||||
m = &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.forge.group.merged", data: []byte(`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`)}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
|
||||
if !strings.Contains(logs.String(), "not a forge message") {
|
||||
t.Errorf("forge/group's merged message read as a group's end: %q", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value)
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
@@ -18,7 +20,7 @@ import (
|
||||
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
|
||||
// (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each
|
||||
// path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk
|
||||
// delivery.Forge.Paths; novox/hq ADR 0499 clause 3).
|
||||
// forgemsg.Forge.Paths; novox/hq ADR 0499 clause 3).
|
||||
const commitTreesTool = "gitea.gitea_commit_trees"
|
||||
|
||||
// toolCaller calls a module's tool, `<module>.<tool>`, as the indexer's own account (mesh-sdk process.Conn.Call).
|
||||
@@ -100,7 +102,7 @@ func filesVia(call toolCaller) func(ctx context.Context, repository, sha string,
|
||||
}
|
||||
for _, d := range dirs {
|
||||
if _, ok := a.Files[d]; !ok {
|
||||
return nil, fmt.Errorf("the answer of %s lists no files of %q", commitTreesTool, d)
|
||||
return nil, fmt.Errorf("the answer of %s lists no files of %q: %w", commitTreesTool, d, index.ErrNotInTree)
|
||||
}
|
||||
}
|
||||
return a.Files, nil
|
||||
@@ -119,6 +121,12 @@ func goModVia(call toolCaller) func(ctx context.Context, repository, sha, dir st
|
||||
return goFileVia(call, "go.mod")
|
||||
}
|
||||
|
||||
// readFileVia reads one file of a repository at a commit by its path (gitea_get_file); a file the commit does not hold is
|
||||
// none. The seat definitions of a revision are read with it (seatsof.go).
|
||||
func readFileVia(call toolCaller) func(ctx context.Context, repository, sha, path string) ([]byte, error) {
|
||||
return goFileVia(call, "")
|
||||
}
|
||||
|
||||
// goFileVia reads one Go file of a module directory at a commit through the forge's get-file tool; a file the commit does
|
||||
// not hold is none.
|
||||
func goFileVia(call toolCaller, name string) func(ctx context.Context, repository, sha, dir string) ([]byte, error) {
|
||||
@@ -128,7 +136,10 @@ func goFileVia(call toolCaller, name string) func(ctx context.Context, repositor
|
||||
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||
}
|
||||
p := name
|
||||
if dir != "" && dir != delivery.RootPath {
|
||||
switch {
|
||||
case name == "": // readFileVia: dir is the file's whole path
|
||||
p = dir
|
||||
case dir != "" && dir != delivery.RootPath:
|
||||
p = dir + "/" + name
|
||||
}
|
||||
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
|
||||
@@ -258,8 +269,8 @@ const (
|
||||
)
|
||||
|
||||
// openVia asks the forge which pull requests into the trunk are open now (gitea_open_pulls), every repository's.
|
||||
func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, error) {
|
||||
return func(context.Context) ([]delivery.OpenPull, error) {
|
||||
func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, error) {
|
||||
return func(context.Context) ([]forgemsg.OpenPull, error) {
|
||||
raw, err := call(openPullsTool, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -275,12 +286,40 @@ func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, er
|
||||
if err := json.Unmarshal(raw, &o); err != nil || o.Pulls == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
|
||||
}
|
||||
out := []delivery.OpenPull{}
|
||||
out := []forgemsg.OpenPull{}
|
||||
for _, p := range *o.Pulls {
|
||||
if p.Base == trunkBranch && p.Number > 0 {
|
||||
out = append(out, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
|
||||
out = append(out, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// openTriesTool is the checker's tool that lists the tries that wait or run (mesh-delivery-checker checker_tries).
|
||||
const openTriesTool = "mesh-delivery-checker.checker_tries"
|
||||
|
||||
// openTriesVia lists the open tries through the checker's tool, never its state (ADR 0596).
|
||||
func openTriesVia(call toolCaller) func(ctx context.Context) ([]tryevent.Open, error) {
|
||||
return func(context.Context) ([]tryevent.Open, error) {
|
||||
raw, err := call(openTriesTool, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// No try open is an answer: the checker's list is empty, or null in an older checker (issue 1921). An answer without the
|
||||
// key is none.
|
||||
var o map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &o); err != nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err)
|
||||
}
|
||||
list, ok := o["tries"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the answer of %s is not {tries}: no tries key", openTriesTool)
|
||||
}
|
||||
var tries []tryevent.Open
|
||||
if err := json.Unmarshal(list, &tries); err != nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err)
|
||||
}
|
||||
return tries, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -50,6 +51,63 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
if _, has := gotArgs.(map[string]any)["paths"]; has {
|
||||
t.Error("the trunk asked for paths")
|
||||
}
|
||||
// A directory the answer lists no files of is a failure of the tree (index.ErrNotInTree); a call that fails is not.
|
||||
noDir := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"files":{}}`), nil }
|
||||
if _, err := filesVia(noDir)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); !errors.Is(err, index.ErrNotInTree) {
|
||||
t.Errorf("a missing directory: %v, want ErrNotInTree", err)
|
||||
}
|
||||
down := func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }
|
||||
if _, err := filesVia(down)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); err == nil || errors.Is(err, index.ErrNotInTree) {
|
||||
t.Errorf("a failed call: %v, want an error that is not ErrNotInTree", err)
|
||||
}
|
||||
// The build the indexer runs is the engine's word (Runs.FP of its composition verb), asked of its own machine.
|
||||
var asked string
|
||||
own := func(key string, body any) (json.RawMessage, error) {
|
||||
asked = key
|
||||
return json.RawMessage(`{"runs":{"fp":"src2:abc"}}`), nil
|
||||
}
|
||||
if fp, err := ownBuildVia(own, "n1")(context.Background()); err != nil || fp != "src2:abc" || asked != "seat:node-engine.composition@n1" {
|
||||
t.Errorf("own build = %q %v asked %q", fp, err, asked)
|
||||
}
|
||||
// Issue 1921: the start during a replacement. What runs still names the old build, the want (composed by the node itself)
|
||||
// the successor, which this process is: it fixes the successor. A want that is not self, or is removed, leaves what runs.
|
||||
for _, row := range []struct{ name, answer, want string }{
|
||||
{"replacing: runs old, want new and self", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true}}`, "src2:new"},
|
||||
{"a want the node does not compose", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new"}}`, "src2:old"},
|
||||
{"replacing: the apply of the want failed or is held", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:new","held":true}}`, "src2:old"},
|
||||
{"replacing: an attempt of another build", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:older"}}`, "src2:new"},
|
||||
{"the want already runs", `{"runs":{"fp":"src2:new"},"want":{"fp":"src2:new","self":true}}`, "src2:new"},
|
||||
{"a removed want", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true,"removed":true}}`, "src2:old"},
|
||||
{"no build runs yet, want self", `{"want":{"fp":"src2:new","self":true}}`, "src2:new"},
|
||||
} {
|
||||
ask := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil }
|
||||
if fp, err := ownBuildVia(ask, "n1")(context.Background()); err != nil || fp != row.want {
|
||||
t.Errorf("%s: own build = %q %v, want %q", row.name, fp, err, row.want)
|
||||
}
|
||||
}
|
||||
none := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }
|
||||
if _, err := ownBuildVia(none, "n1")(context.Background()); err == nil {
|
||||
t.Error("an engine that says nothing runs gave a build")
|
||||
}
|
||||
// Issue 1921: the checker's list of open tries; none open is an answer, whether [] or null.
|
||||
for _, row := range []struct {
|
||||
name, answer string
|
||||
want int
|
||||
refused bool
|
||||
}{
|
||||
{"one open", `{"tries":[{"id":"try-1","state":"running","whole":true,"members":[{"repository":"novox/a","head":"h","trunk":"t"}]}]}`, 1, false},
|
||||
{"an empty list", `{"tries":[]}`, 0, false},
|
||||
{"null, as an older checker answers none", `{"tries":null}`, 0, false},
|
||||
{"no tries key", `{}`, 0, true},
|
||||
{"not json", `nope`, 0, true},
|
||||
{"tries of another shape", `{"tries":{"a":1}}`, 0, true},
|
||||
} {
|
||||
call := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil }
|
||||
got, err := openTriesVia(call)(context.Background())
|
||||
if (err != nil) != row.refused || len(got) != row.want {
|
||||
t.Errorf("%s: %d tries, error %v; want %d, refused %v", row.name, len(got), err, row.want, row.refused)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// compositionVerb is the node-engine seat's read verb that says what runs of a module on a machine (its Runs.FP), asked of
|
||||
// that machine's own engine as `seat:node-engine.composition@<machine>`. The indexer learns the build it runs from it, never
|
||||
// from the trunk's value of itself or the haves bucket.
|
||||
const compositionVerb = "seat:node-engine.composition"
|
||||
|
||||
// ownBuildVia asks the engine of node which fingerprint of the indexer runs there.
|
||||
func ownBuildVia(call toolCaller, node string) func(ctx context.Context) (string, error) {
|
||||
return func(context.Context) (string, error) {
|
||||
raw, err := call(compositionVerb+"@"+node, map[string]any{"module": delivery.IndexerModule})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s: %w", compositionVerb, err)
|
||||
}
|
||||
var a struct {
|
||||
Runs *struct {
|
||||
FP string `json:"fp"`
|
||||
} `json:"runs"`
|
||||
Want *struct {
|
||||
FP string `json:"fp"`
|
||||
Self bool `json:"self"`
|
||||
Removed bool `json:"removed"`
|
||||
} `json:"want"`
|
||||
// Attempt is the engine's last apply that did not end running (failed, or held), with the build it was of.
|
||||
Attempt *struct {
|
||||
FP string `json:"fp"`
|
||||
} `json:"attempt"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return "", fmt.Errorf("the answer of %s is not a composition: %w", compositionVerb, err)
|
||||
}
|
||||
// While the engine's apply of the want is in progress (what runs still names the old build, the want names another, and no
|
||||
// attempt names that want as failed or held), this process is the successor and fixes the want's build. self is the node's
|
||||
// standing way of composing the module, not a mark of this process, so it is not enough alone: an old process started again
|
||||
// while the want names a successor that does not run (its apply failed or is held) takes what runs. The start fixes the
|
||||
// build once (quick fix, ADR 0639 §7: nothing is learnt later, nothing waits on a timer); the thorough form is the
|
||||
// engine passing each process the fingerprint it starts it with.
|
||||
running := ""
|
||||
if a.Runs != nil {
|
||||
running = a.Runs.FP
|
||||
}
|
||||
if w := a.Want; w != nil && w.Self && !w.Removed && w.FP != "" && w.FP != running && (a.Attempt == nil || a.Attempt.FP != w.FP) {
|
||||
return w.FP, nil
|
||||
}
|
||||
if a.Runs == nil || a.Runs.FP == "" {
|
||||
return "", errors.New("the engine says no fingerprint of the indexer runs")
|
||||
}
|
||||
return a.Runs.FP, nil
|
||||
}
|
||||
}
|
||||
+8
-7
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -71,10 +72,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
|
||||
}
|
||||
serveForge(t, js.Conn(), listing, goSumContent)
|
||||
})
|
||||
merged := func(repo string, number int, merge string, f delivery.Forge) {
|
||||
merged := func(repo string, number int, merge string, f forgemsg.Forge) {
|
||||
t.Helper()
|
||||
f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = delivery.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge
|
||||
data, err := delivery.EncodeForge(f)
|
||||
f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = forgemsg.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge
|
||||
data, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -101,10 +102,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
|
||||
"tsconfig.json": "5555555555555555555555555555555555555555"}}
|
||||
}
|
||||
// mesh-sdk at m1, then the catalogue's two modules: their go.sum names mesh-sdk, read by the indexer itself.
|
||||
merged("novox/mesh-sdk", 70, m1, delivery.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1},
|
||||
merged("novox/mesh-sdk", 70, m1, forgemsg.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1},
|
||||
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("6666666666666666666666666666666666666666")})
|
||||
module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m1 })
|
||||
merged("novox/mesh-catalog", 301, catalogMerge, delivery.Forge{Root: "r0",
|
||||
merged("novox/mesh-catalog", 301, catalogMerge, forgemsg.Forge{Root: "r0",
|
||||
Trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"},
|
||||
Manifests: map[string]string{"modules/importer": named("importer"), "modules/plain": named("plain")},
|
||||
Locks: map[string]map[string]string{"modules/importer": {"go.sum": goSumHash}, "modules/plain": {"go.sum": goSumHash}}})
|
||||
@@ -140,11 +141,11 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
|
||||
// over its named tree, is not written (issue 1758, ADR 0499 clause 6; the operator, 2026-10-25: a module's value moves
|
||||
// only with the module).
|
||||
const goOnly = "4444444444444444444444444444444444444444"
|
||||
merged("novox/mesh-sdk", 72, goOnly, delivery.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc",
|
||||
merged("novox/mesh-sdk", 72, goOnly, forgemsg.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc",
|
||||
Trees: map[string]string{"": "cccccccccccccccccccccccccccccccccccccccc"}, Manifests: map[string]string{"": string(sdkManifest)},
|
||||
Paths: sdkObjects("6666666666666666666666666666666666666666")})
|
||||
// mesh-sdk merges a change of buildstep alone (its version moves: src changed too).
|
||||
merged("novox/mesh-sdk", 71, m2, delivery.Forge{Root: sdk2, Trees: map[string]string{"": sdk2},
|
||||
merged("novox/mesh-sdk", 71, m2, forgemsg.Forge{Root: sdk2, Trees: map[string]string{"": sdk2},
|
||||
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("7777777777777777777777777777777777777777")})
|
||||
sdkAt2 := module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m2 })
|
||||
if sdkAt2.Inputs2 == nil || !strings.HasPrefix(sdkAt2.Inputs2.Tree, delivery.NamedTreePrefix) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -73,7 +74,10 @@ func (w *previewWatch) ensure() error {
|
||||
}
|
||||
})
|
||||
}()
|
||||
return w.ix.RecomputeCandidates(w.ctx)
|
||||
if err := w.ix.RecomputeCandidates(w.ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.ix.WriteTryCandidates(w.ctx)
|
||||
}
|
||||
|
||||
// Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the
|
||||
@@ -90,3 +94,44 @@ func (w *previewWatch) Written(m delivered) {
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// tryEventSubject is the subject of an event of the checker's stream checks about a try.
|
||||
func tryEventSubject(event string) string {
|
||||
return "mesh.stream." + delivery.CheckerModule + ".event." + event
|
||||
}
|
||||
|
||||
// heardChecks takes one event of the checker's stream in: a preview written (Written), or a whole try's start or end, from
|
||||
// which the candidate of its group is written or deleted (index.Indexer.StartTry, EndTry). A failure stays unacknowledged, and
|
||||
// the server delivers it again; a body that is no try event is terminated.
|
||||
func (w *previewWatch) heardChecks(m delivered) {
|
||||
var event string
|
||||
switch m.Subject() {
|
||||
case tryEventSubject(tryevent.Started):
|
||||
event = tryevent.Started
|
||||
case tryEventSubject(tryevent.Ended):
|
||||
event = tryevent.Ended
|
||||
default:
|
||||
w.Written(m)
|
||||
return
|
||||
}
|
||||
e, err := tryevent.Decode(event, m.Data())
|
||||
if err != nil {
|
||||
w.logger.Printf("%s: not a try event, dropped: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if event == tryevent.Started {
|
||||
var ms []index.TryMember
|
||||
for _, x := range e.Members {
|
||||
ms = append(ms, index.TryMember{Repository: x.Repository, Head: x.Head, Trunk: x.Trunk})
|
||||
}
|
||||
err = w.ix.StartTry(w.ctx, e.ID, ms)
|
||||
} else {
|
||||
err = w.ix.EndTry(w.ctx, e.ID)
|
||||
}
|
||||
if err != nil {
|
||||
w.logger.Printf("%s of %s: left for redelivery: %v", event, e.ID, err)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
@@ -14,6 +14,13 @@ import (
|
||||
// takerIn is the part of the indexer a takeins value asks through.
|
||||
type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
TakeInRepository(ctx context.Context, repository string) index.TakenIn
|
||||
JudgeAgain(ctx context.Context, module, digest string) index.TakenIn
|
||||
TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn
|
||||
DropPending(ctx context.Context, key, by, why string) index.TakenIn
|
||||
TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn
|
||||
DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn
|
||||
RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
@@ -31,9 +38,32 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
if !a.Answered.IsZero() {
|
||||
return
|
||||
}
|
||||
r := ix.TakeIn(ctx, a.Module)
|
||||
asked, r := "take-in", index.TakenIn{}
|
||||
if a.RecoverTries {
|
||||
asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why)
|
||||
a.Module = "tries"
|
||||
} else if a.TryCandidate != "" && a.Drop {
|
||||
asked, r = "drop of a try's candidate", ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why)
|
||||
a.Module = a.TryCandidate
|
||||
} else if a.Merge != "" {
|
||||
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
|
||||
a.Module = r.Module
|
||||
} else if a.Pending != "" && a.Drop {
|
||||
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why)
|
||||
a.Module = a.Pending
|
||||
} else if a.Pending != "" {
|
||||
asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why)
|
||||
a.Module = a.Pending
|
||||
} else if a.Repository != "" {
|
||||
asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository)
|
||||
a.Module = a.Repository
|
||||
} else if a.Judge {
|
||||
asked, r = "judging again", ix.JudgeAgain(ctx, a.Module, a.Digest)
|
||||
} else {
|
||||
r = ix.TakeIn(ctx, a.Module)
|
||||
}
|
||||
a.Result, a.Answered = &r, time.Now().UTC()
|
||||
logf("%s: a take-in asked by %s (%s): %s", a.Module, a.By, a.Why, r.Answer)
|
||||
logf("%s: a %s asked by %s (%s): %s", a.Module, asked, a.By, a.Why, r.Answer)
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -36,3 +37,54 @@ func sayRefused(js jetstream.JetStream, cred process.Account, logf func(string,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayAccepted says the indexer's event manifest-accepted (issue 1966 D3, novox/hq ADR 0595 §4):
|
||||
// mesh.mod.<module>.event.manifest-accepted, carrying the module's bus-state entries whole, with the digest and judge as
|
||||
// the bus's dedup id (delivery.ManifestAccepted.EventID), so one acceptance is said once.
|
||||
func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string, ...any)) func(context.Context, delivery.ManifestAccepted) {
|
||||
return func(ctx context.Context, a delivery.ManifestAccepted) {
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
logf("judge: %s for %s is not said: %v", delivery.ManifestAcceptedEvent, a.Module, err)
|
||||
return
|
||||
}
|
||||
msg := nats.NewMsg(cred.EventSubject(delivery.ManifestAcceptedEvent))
|
||||
id := a.EventID()
|
||||
msg.Header.Set("x-event-id", id)
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
if _, err := js.PublishMsg(pub, msg, jetstream.WithMsgID(id)); err != nil {
|
||||
logf("judge: %s", fmt.Errorf("saying %s for %s: %w", delivery.ManifestAcceptedEvent, a.Module, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events
|
||||
// take: mesh.mod.<module>.event.take-in.<kind>, the wait's key in the body. A publish that fails is the caller's to say.
|
||||
func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error {
|
||||
return func(ctx context.Context, t takein.TakeIn) error {
|
||||
event, err := takein.TakeInEvent(t.Kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := takein.EncodeTakeIn(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(cred.EventSubject(event))
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = js.PublishMsg(pub, msg)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user