Compare commits
57
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b1094a9265 | ||
|
|
4907470e90 | ||
|
|
55fb32ba25 | ||
|
|
cd01f1c196 | ||
|
|
261f8d54f7 | ||
|
|
643ec7cd91 | ||
|
|
c0e5bc2834 | ||
|
|
6f048a239c | ||
|
|
fab533dfd0 | ||
|
|
39285c95bd | ||
|
|
1b262125ee | ||
|
|
bfe4caf196 | ||
|
|
3742a7c54c | ||
|
|
f1e6520037 | ||
|
|
4ddf78395d | ||
|
|
8c67adaf91 | ||
|
|
e3735da928 | ||
|
|
78d57e1cc6 | ||
|
|
9f9e743b2f | ||
|
|
abe9ea3796 | ||
|
|
f9578cb7b4 | ||
|
|
69a2790099 | ||
|
|
88a47be251 | ||
|
|
bd5eb33e9e | ||
|
|
c2b8ac3370 | ||
|
|
92c38981ea | ||
|
|
45386212da | ||
|
|
b97e277d2d | ||
|
|
8823cc29d6 | ||
|
|
55936c1cd7 | ||
|
|
31ff708f88 | ||
|
|
2fcd61baa6 | ||
|
|
655791b8a3 | ||
|
|
3c832d1a2d | ||
|
|
0605962960 | ||
|
|
343321f11e | ||
|
|
4b7a31b2c7 | ||
|
|
4596fbe1b7 | ||
|
|
c78941e80b | ||
|
|
7b7096f37e | ||
|
|
4e4601a1eb | ||
|
|
a395b49be2 | ||
|
|
1040c8ec46 | ||
|
|
92dc5f03d0 | ||
|
|
ef13e63268 | ||
|
|
9c6e70e555 | ||
|
|
ad9a353484 | ||
|
|
a762701e3d | ||
|
|
d6b6f24e98 | ||
|
|
ed4f07b0d8 | ||
|
|
7609f2d9a2 | ||
|
|
d11a5af113 | ||
|
|
531d05ab4b | ||
|
|
071a28f218 | ||
|
|
72633722c5 | ||
|
|
0579e4e74d | ||
|
|
2cb3198049 |
@@ -62,8 +62,6 @@ type fakeBuildModules struct {
|
||||
js jetstream.JetStream
|
||||
ctx context.Context
|
||||
failBuild atomic.Bool
|
||||
// failRetakeable makes every build fail as a lookup that timed out (delivery.Retake), which passes by itself.
|
||||
failRetakeable atomic.Bool
|
||||
// refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2).
|
||||
refusePush atomic.Bool
|
||||
mu sync.Mutex
|
||||
@@ -119,8 +117,6 @@ func (f *fakeBuildModules) step(module, name string, args map[string]any) (any,
|
||||
merge = `,"merge":"` + strings.Repeat("9", 40) + `"`
|
||||
}
|
||||
return json.RawMessage(`{"workspace":"w` + buildstep.NewID() + `","commit":"` + commit + `"` + merge + `}`), nil
|
||||
case f.failRetakeable.Load() && tool != "build_image_copy":
|
||||
return nil, fmt.Errorf("%slookup git.novox.be on 10.10.0.3:53: i/o timeout", delivery.Retake)
|
||||
case f.failBuild.Load() && tool != "build_image_copy": // the build fails wherever it runs
|
||||
return nil, fmt.Errorf("go build of %v: boom", args["artifact"])
|
||||
case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4)
|
||||
@@ -495,47 +491,30 @@ func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) {
|
||||
// failed naming it; the indexer writing modules.<m> again with the same fingerprint wakes the builder, which builds
|
||||
// the key again as a new revision of the same key, now done.
|
||||
func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7
|
||||
t.Run("a new modules value with the same fingerprint", func(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.fake.failBuild.Store(true)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildFailed))
|
||||
if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") {
|
||||
t.Errorf("build = %+v, want failed naming the node and the step's reason", v)
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey("ca-trust", fp)
|
||||
failed, err := b.builds.Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.fake.failBuild.Store(false)
|
||||
// The indexer writes the module again (here at a later commit), with the same fingerprint.
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef")
|
||||
b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
done, err := b.builds.Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if done.Revision() <= failed.Revision() {
|
||||
t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision())
|
||||
}
|
||||
})
|
||||
// 1748/AC6: a build failed retakeable is built again after its backoff, with no new modules value or other event, and
|
||||
// ends done; the key says it was retaken.
|
||||
t.Run("a retakeable failure after its backoff", func(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.fake.failRetakeable.Store(true)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", map[string]string{"MESH_BUILDER_RETAKE_BACKOFF": "1s"})
|
||||
if v := b.await(t, "ca-trust", isState(delivery.BuildFailed)); !delivery.Retakeable(v.Error) {
|
||||
t.Fatalf("build = %+v, want failed retakeable", v)
|
||||
}
|
||||
b.fake.failRetakeable.Store(false) // the lookup passes
|
||||
if v := b.await(t, "ca-trust", isState(delivery.BuildDone)); v.Retakes < 1 {
|
||||
t.Errorf("done key = %+v, want its retake counted", v)
|
||||
}
|
||||
})
|
||||
b := startBed(t)
|
||||
b.fake.failBuild.Store(true)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildFailed))
|
||||
if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") {
|
||||
t.Errorf("build = %+v, want failed naming the node and the step's reason", v)
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey("ca-trust", fp)
|
||||
failed, err := b.builds.Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.fake.failBuild.Store(false)
|
||||
// The indexer writes the module again (here at a later commit), with the same fingerprint.
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef")
|
||||
b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
done, err := b.builds.Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if done.Revision() <= failed.Revision() {
|
||||
t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision())
|
||||
}
|
||||
}
|
||||
|
||||
// TestAToolchainChangeDoesNotBuildTheOldFingerprint: the fingerprint names go1.27.1, the nodes then agree on go1.27.2 (a
|
||||
|
||||
@@ -77,12 +77,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err)
|
||||
}
|
||||
}
|
||||
var retakeBackoff time.Duration // 0 is build.DefaultRetakeBackoff
|
||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_RETAKE_BACKOFF")); v != "" {
|
||||
if retakeBackoff, err = time.ParseDuration(v); err != nil {
|
||||
return fmt.Errorf("MESH_BUILDER_RETAKE_BACKOFF: %w", err)
|
||||
}
|
||||
}
|
||||
tools := &buildrun.ToolchainSet{}
|
||||
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||
@@ -127,8 +121,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Logf: logger.Printf,
|
||||
Stored: storedVia(calls.Call),
|
||||
Node: cred.Node,
|
||||
// A key failed retakeable (delivery.Retake) is taken again by itself after this wait, at most delivery.MaxRetakes times (issue 1748).
|
||||
RetakeBackoff: retakeBackoff,
|
||||
}
|
||||
// A candidate's preview is merged onto the trunk head the indexer previewed it onto: source.main at the candidate's base
|
||||
// revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104214149-0d97c09c49b4
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104214149-0d97c09c49b4 h1:K2niszrlnJ7JMiXI6xg4gG90YCd/tqNLGBvujLInI48=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104214149-0d97c09c49b4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -66,10 +66,6 @@ type ClaimOptions struct {
|
||||
// done: the store's nightly collector deletes a blob nothing holds, and a key the controller never took in holds
|
||||
// nothing, so a key whose blob is gone is claimed building as a new revision and built again. Nil asks nothing.
|
||||
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||
// Retaking is this claim being the builder's backoff taking a retakeable failed key again (Builder.armRetake, issue
|
||||
// 1748): only it adds to BuilderBuild.Retakes, so the cap of delivery.MaxRetakes counts the backoff's own retakes and
|
||||
// not a look another event makes while the backoff waits.
|
||||
Retaking bool
|
||||
}
|
||||
|
||||
// StoredReferences are the references of a done build's bundles and archives that name a blob in the artifact store
|
||||
@@ -157,8 +153,7 @@ func Unreferenced(b delivery.BuilderBuild) bool {
|
||||
// - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no
|
||||
// builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478
|
||||
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
|
||||
// which wait for another fingerprint (a retakeable failure, delivery.Retakeable, is taken again too; the new
|
||||
// revision keeps Retakes, and counts one more only when it is the backoff's claim, ClaimOptions.Retaking), and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
|
||||
// which wait for another fingerprint, and one failed for want of an image builder (NoImageBuilderPrefix), which waits for
|
||||
// another fingerprint or a node that runs the image build module (ClaimOptions.ImageBuilder);
|
||||
// - a key queued waiting for a package version (Waits) is claimed building as a new revision once that version is
|
||||
// published (ClaimOptions.Published, ADR 0500 clause 6), and not before;
|
||||
@@ -217,10 +212,6 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
|
||||
case cur.State == delivery.BuildFailed && strings.HasPrefix(cur.Error, NoImageBuilderPrefix) && (opt.ImageBuilder == nil || !opt.ImageBuilder()):
|
||||
return nil, nil
|
||||
case cur.State == delivery.BuildFailed:
|
||||
building.Retakes = cur.Retakes // the count rides on the key's value, so it survives a restart (issue 1748)
|
||||
if opt.Retaking && delivery.Retakeable(cur.Error) {
|
||||
building.Retakes++
|
||||
}
|
||||
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
|
||||
case cur.State == delivery.BuildQueued && cur.Waits != "":
|
||||
if opt.Published == nil || !opt.Published(cur.Waits) {
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// clock is the builder's fake clock for the retake backoff (issue 1748): After records what is asked and fire runs it.
|
||||
type clock struct {
|
||||
mu sync.Mutex
|
||||
waits []time.Duration
|
||||
pending []func()
|
||||
}
|
||||
|
||||
func (c *clock) after(d time.Duration, f func()) func() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.waits, c.pending = append(c.waits, d), append(c.pending, f)
|
||||
return func() {}
|
||||
}
|
||||
|
||||
// fire runs every timer set so far, once, and answers how many there were.
|
||||
func (c *clock) fire() int {
|
||||
c.mu.Lock()
|
||||
fs := c.pending
|
||||
c.pending = nil
|
||||
c.mu.Unlock()
|
||||
for _, f := range fs {
|
||||
f()
|
||||
}
|
||||
return len(fs)
|
||||
}
|
||||
|
||||
func retakeRig(t *testing.T) (*rig, *clock) {
|
||||
r := newRig(t, goBundle)
|
||||
c := &clock{}
|
||||
r.b.RetakeBackoff, r.b.After = 10*time.Second, c.after
|
||||
return r, c
|
||||
}
|
||||
|
||||
const lookupFailure = delivery.Retake + "go mod download: lookup git.novox.be on 10.10.0.3:53: i/o timeout"
|
||||
|
||||
// 1748: a key failed retakeable is taken again by the builder's backoff, and only by it counts toward the cap.
|
||||
func TestTheBuilderRetakesARetakeableFailure(t *testing.T) {
|
||||
// AC3: claimed again as a new revision after the backoff, with no other event; the count kept on the key stops it after
|
||||
// delivery.MaxRetakes, and a builder that starts again reads the count.
|
||||
t.Run("after its backoff, at most three times", func(t *testing.T) {
|
||||
r, c := retakeRig(t)
|
||||
r.bm.fails["ace/build_go_build"] = lookupFailure
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildFailed || !delivery.Retakeable(v.Error) || v.Retakes != 0 {
|
||||
t.Fatalf("build = %+v, want failed retakeable with no retake yet", v)
|
||||
}
|
||||
if len(c.waits) != 1 || c.waits[0] != 10*time.Second {
|
||||
t.Fatalf("backoffs = %v, want one of 10s", c.waits)
|
||||
}
|
||||
rev := r.st.revs[r.key]
|
||||
for retake := 1; retake <= delivery.MaxRetakes; retake++ {
|
||||
if c.fire() != 1 {
|
||||
t.Fatalf("retake %d: no timer was set", retake)
|
||||
}
|
||||
r.b.Wait()
|
||||
v = valueOf(t, r.st, r.key)
|
||||
if v.Retakes != retake || r.st.revs[r.key] <= rev {
|
||||
t.Fatalf("retake %d: key = %+v at revision %d (was %d)", retake, v, r.st.revs[r.key], rev)
|
||||
}
|
||||
rev = r.st.revs[r.key]
|
||||
}
|
||||
if len(c.pending) != 0 || len(c.waits) != delivery.MaxRetakes || c.waits[1] != 20*time.Second || c.waits[2] != 40*time.Second {
|
||||
t.Fatalf("after %d retakes: %d pending, backoffs %v; want none pending and 10s, 20s, 40s", delivery.MaxRetakes, len(c.pending), c.waits)
|
||||
}
|
||||
if n := len(r.bm.callsOf("build_go_build")); n != 1+delivery.MaxRetakes {
|
||||
t.Errorf("%d builds, want the first and %d retakes", n, delivery.MaxRetakes)
|
||||
}
|
||||
// A builder that starts again reads the count from the key: at the cap it arms nothing, below it the next wait.
|
||||
restarted := func(retakes int) *clock {
|
||||
rr, cc := retakeRig(t)
|
||||
rr.st, rr.b.Claimer.Store = r.st, r.st
|
||||
data, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildFailed, Error: lookupFailure, Retakes: retakes})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r.st.vals[r.key] = data
|
||||
rr.b.init()
|
||||
rr.b.known["m"] = delivery.Module{FP2: fp1, Manifest: mdigest('a')}
|
||||
if err := rr.b.Sweep(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cc
|
||||
}
|
||||
if cc := restarted(delivery.MaxRetakes); len(cc.waits) != 0 {
|
||||
t.Errorf("a key retaken %d times was armed again: %v", delivery.MaxRetakes, cc.waits)
|
||||
}
|
||||
if cc := restarted(1); len(cc.waits) != 1 || cc.waits[0] != 20*time.Second {
|
||||
t.Errorf("a key retaken once was armed with %v, want one wait of 20s", cc.waits)
|
||||
}
|
||||
})
|
||||
// AC4: a failure of the source (a compile error, FailedUntilChanged) is never taken again by the backoff.
|
||||
t.Run("a failure of the source is never taken again", func(t *testing.T) {
|
||||
for _, reason := range []string{"go build of tool: undefined: x", delivery.NotPinned + "up: names no digest", delivery.VersionTaken + "@x/y@1 holds another file"} {
|
||||
r, c := retakeRig(t)
|
||||
r.bm.fails["ace/build_go_build"] = reason
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildFailed || delivery.Retakeable(v.Error) {
|
||||
t.Fatalf("%q: build = %+v, want failed and not retakeable", reason, v)
|
||||
}
|
||||
if len(c.waits) != 0 || c.fire() != 0 {
|
||||
t.Errorf("%q: the backoff was armed: %v", reason, c.waits)
|
||||
}
|
||||
}
|
||||
})
|
||||
// Only the backoff's claim counts: a look another event makes while the backoff waits (a toolchain change, a new modules
|
||||
// value) takes the key again and leaves the count as it was, so those looks cannot spend the retakes inside the pause.
|
||||
t.Run("another event's look leaves the count", func(t *testing.T) {
|
||||
r, c := retakeRig(t)
|
||||
r.bm.fails["ace/build_go_build"] = lookupFailure
|
||||
r.consider(t, "modules/m")
|
||||
for i := 0; i < 2*delivery.MaxRetakes; i++ {
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.Retakes != 0 || v.State != delivery.BuildFailed {
|
||||
t.Fatalf("after Reconsider looks the key = %+v, want failed with Retakes 0", v)
|
||||
}
|
||||
if c.fire() != 1 {
|
||||
t.Fatal("the backoff's own wait was not armed once")
|
||||
}
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.Retakes != 1 {
|
||||
t.Errorf("after the backoff's claim the key = %+v, want Retakes 1", v)
|
||||
}
|
||||
})
|
||||
// The journal says each retake with its count and the log's entry of the build's end names it.
|
||||
t.Run("the journal and the log say the count", func(t *testing.T) {
|
||||
r, c := retakeRig(t)
|
||||
var said []string
|
||||
r.b.Logf = func(f string, a ...any) { said = append(said, strings.TrimSpace(fmt.Sprintf(f, a...))) }
|
||||
var ended []delivery.BuildLogEntry
|
||||
r.b.Node, r.b.Log = "novox", func(e delivery.BuildLogEntry) {
|
||||
if e.Event == delivery.BuildEndedEvent {
|
||||
ended = append(ended, e)
|
||||
}
|
||||
}
|
||||
r.bm.fails["ace/build_go_build"] = lookupFailure
|
||||
r.consider(t, "modules/m")
|
||||
c.fire()
|
||||
r.b.Wait()
|
||||
if len(ended) != 2 || ended[0].Retakes != 0 || ended[1].Retakes != 1 {
|
||||
t.Fatalf("ended entries = %+v, want retakes 0 then 1; journal: %v", ended, said)
|
||||
}
|
||||
if joined := strings.Join(said, "\n"); !strings.Contains(joined, "taken again after its backoff (retake 1 of 3)") {
|
||||
t.Errorf("the journal says %q, want the retake with its count", joined)
|
||||
}
|
||||
})
|
||||
// The key's error opens with the retake prefix only when every failing lane failed retakeable.
|
||||
t.Run("the key error opens with the prefix only when every lane is retakeable", func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
errs []string
|
||||
prefix bool
|
||||
}{
|
||||
{"one retakeable lane", []string{"mesh-build-go on ace: the step failed: " + lookupFailure}, true},
|
||||
{"two retakeable lanes", []string{"a: " + lookupFailure, "b: " + lookupFailure}, true},
|
||||
{"a retakeable and a compile error", []string{"a: " + lookupFailure, "b: go build of tool: undefined: x"}, false},
|
||||
{"a compile error", []string{"b: go build of tool: undefined: x"}, false},
|
||||
} {
|
||||
got := keyError(c.errs)
|
||||
if has := strings.HasPrefix(got, delivery.Retake); has != c.prefix || delivery.Retakeable(got) != c.prefix {
|
||||
t.Errorf("%s: keyError = %q, want prefix %v", c.name, got, c.prefix)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -79,12 +79,6 @@ type Builder struct {
|
||||
// time.Now. Node is the node the builder runs on, which asked the build.
|
||||
Now func() time.Time
|
||||
Node string
|
||||
// RetakeBackoff is the wait before the builder takes a key that failed retakeable (delivery.Retakeable) again by itself,
|
||||
// doubled with each retake (issue 1748): DefaultRetakeBackoff when 0. After runs f once d has passed and returns a
|
||||
// stop; nil waits a context's bound (ADR 0437 §8: the builder holds no timer). The wait is re-armed from the key's value at
|
||||
// start (Sweep): a build's work is not a message of the builder's own on a stream, so there is no delivery delay to ride.
|
||||
RetakeBackoff time.Duration
|
||||
After func(d time.Duration, f func()) (stop func())
|
||||
|
||||
once sync.Once
|
||||
sem chan struct{}
|
||||
@@ -108,14 +102,8 @@ type Builder struct {
|
||||
waiting map[string]map[string]chan string
|
||||
// hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key.
|
||||
hands map[string]*handWatch
|
||||
// armed are the keys with a retake wait running, by build key; retaking are the keys whose backoff has passed and whose
|
||||
// next claim is the backoff's own (ClaimOptions.Retaking).
|
||||
armed, retaking map[string]bool
|
||||
}
|
||||
|
||||
// DefaultRetakeBackoff is the first wait before a retakeable failure is taken again; each retake doubles it.
|
||||
const DefaultRetakeBackoff = 60 * time.Second
|
||||
|
||||
type planned struct {
|
||||
plan Plan
|
||||
err error
|
||||
@@ -132,7 +120,6 @@ func (b *Builder) init() {
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||
b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{}
|
||||
b.hands = map[string]*handWatch{}
|
||||
b.armed, b.retaking = map[string]bool{}, map[string]bool{}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -200,14 +187,6 @@ func (b *Builder) Sweep(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
newly = b.notePublished(cur) || newly
|
||||
if current[key] && cur.State == delivery.BuildFailed {
|
||||
if module, _, err := delivery.ParseBuilderBuildKey(key); err == nil {
|
||||
b.mu.Lock()
|
||||
fp := b.known[module].FP2
|
||||
b.mu.Unlock()
|
||||
b.armRetake(ctx, key, module, fp, cur) // a restarted builder reads the count and goes on
|
||||
}
|
||||
}
|
||||
if current[key] || cur.State != delivery.BuildBuilding {
|
||||
continue
|
||||
}
|
||||
@@ -450,11 +429,9 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
trunk := b.trunkFP[key]
|
||||
first := !b.tried[key]
|
||||
b.tried[key] = true
|
||||
retaking := b.retaking[key]
|
||||
delete(b.retaking, key)
|
||||
b.mu.Unlock()
|
||||
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
|
||||
Stored: b.storedOnce(key), Retaking: retaking})
|
||||
Stored: b.storedOnce(key)})
|
||||
if err != nil {
|
||||
b.logf("%s: %v", module, err)
|
||||
return
|
||||
@@ -500,7 +477,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
b.logf("%s: left building, the toolchains moved under it", cl.Key)
|
||||
return
|
||||
}
|
||||
result.By, result.Manifest, result.Retakes = b.Claimer.By, m.Manifest, cl.Build.Retakes
|
||||
result.By, result.Manifest = b.Claimer.By, m.Manifest
|
||||
if len(cl.Gone) > 0 && result.State == delivery.BuildDone {
|
||||
result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")"
|
||||
}
|
||||
@@ -516,7 +493,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
b.logBuild(LogEnded(module, m, b.refOf(module, m.FP2), b.Node, b.now(), result))
|
||||
if result.State == delivery.BuildFailed {
|
||||
b.logf("%s: failed: %s", cl.Key, result.Error)
|
||||
b.armRetake(ctx, cl.Key, module, m.FP2, result)
|
||||
}
|
||||
if result.State == delivery.BuildQueued {
|
||||
b.logf("%s: waits for the package %s to be published", cl.Key, result.Waits)
|
||||
@@ -527,57 +503,6 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
}
|
||||
}
|
||||
|
||||
// armRetake takes a key that failed retakeable again by itself once its backoff has passed, as a new revision (issue 1748):
|
||||
// only an error that opens with delivery.Retake, and at most delivery.MaxRetakes times, counted on the key's value. A key
|
||||
// failed for its source (a compile error, FailedUntilChanged) is never armed.
|
||||
func (b *Builder) armRetake(ctx context.Context, key, module, fp string, v delivery.BuilderBuild) {
|
||||
if v.State != delivery.BuildFailed || !delivery.Retakeable(v.Error) || v.Retakes >= delivery.MaxRetakes || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
if b.armed[key] {
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.armed[key] = true
|
||||
b.mu.Unlock()
|
||||
wait := b.RetakeBackoff
|
||||
if wait <= 0 {
|
||||
wait = DefaultRetakeBackoff
|
||||
}
|
||||
wait <<= v.Retakes
|
||||
b.logf("%s: failed retakeable; taken again in %s (retake %d of %d)", key, wait, v.Retakes+1, delivery.MaxRetakes)
|
||||
after := b.After
|
||||
if after == nil {
|
||||
// The builder holds no timer (ADR 0437 §8: a wait is a context's bound): the wait is the bound of a context, which
|
||||
// the builder's own context ends early.
|
||||
after = func(d time.Duration, f func()) func() {
|
||||
wait, stop := context.WithTimeout(ctx, d)
|
||||
go func() {
|
||||
<-wait.Done()
|
||||
if errors.Is(wait.Err(), context.DeadlineExceeded) && ctx.Err() == nil {
|
||||
f()
|
||||
}
|
||||
}()
|
||||
return stop
|
||||
}
|
||||
}
|
||||
after(wait, func() {
|
||||
b.mu.Lock()
|
||||
delete(b.armed, key)
|
||||
m, known := b.known[module]
|
||||
b.mu.Unlock()
|
||||
if ctx.Err() != nil || !known || m.FP2 != fp {
|
||||
return // a newer fingerprint supersedes it: that one is built by its own event
|
||||
}
|
||||
b.logf("%s: taken again after its backoff (retake %d of %d)", key, v.Retakes+1, delivery.MaxRetakes)
|
||||
b.mu.Lock()
|
||||
b.retaking[key] = true
|
||||
b.mu.Unlock()
|
||||
b.consider(ctx, module, m)
|
||||
})
|
||||
}
|
||||
|
||||
// stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan
|
||||
// has agreed (novox/hq ADR 0465) must be the one the fingerprint was computed with (Module.Toolchains).
|
||||
func (b *Builder) stale(m delivery.Module, plan Plan) string {
|
||||
@@ -799,19 +724,6 @@ func keyError(errs []string) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
// A retakeable failure opens the key's error only when every failing lane is one (issue 1748): a lane that failed on its
|
||||
// source would be built again for nothing.
|
||||
all := len(errs) > 0
|
||||
for _, e := range errs {
|
||||
all = all && strings.Contains(e, delivery.Retake)
|
||||
}
|
||||
if all {
|
||||
stripped := make([]string, len(errs))
|
||||
for i, e := range errs {
|
||||
stripped[i] = strings.Replace(e, delivery.Retake, "", 1)
|
||||
}
|
||||
return delivery.Retake + strings.Join(stripped, "; ")
|
||||
}
|
||||
return strings.Join(errs, "; ")
|
||||
}
|
||||
|
||||
@@ -1043,7 +955,6 @@ func LogEnded(module string, m delivery.Module, ref, by string, at time.Time, re
|
||||
e.Against = append([]string(nil), m.Inputs2.Bases...)
|
||||
}
|
||||
e.Made = append([]delivery.BuiltArtifact(nil), result.Artifacts...)
|
||||
e.Retakes = result.Retakes
|
||||
e.Imports = append([]string(nil), result.Imports...) // the indexer keeps them on the module (novox/hq ADR 0593 §2)
|
||||
return e
|
||||
}
|
||||
|
||||
@@ -212,25 +212,4 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR
|
||||
if files < 1 {
|
||||
t.Fatalf("read %d source files, want the program's", files)
|
||||
}
|
||||
// The one bounded wait the builder holds for its own sake is the retake backoff (issue 1748): a goroutine that waits a
|
||||
// context's deadline and then acts. It lives in Builder.armRetake only; any other source holding that idiom is refused.
|
||||
const idiom = "context.DeadlineExceeded"
|
||||
var holders []string
|
||||
for _, root := range []string{"cmd", "internal"} {
|
||||
filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
src, _ := os.ReadFile(path)
|
||||
for _, fn := range strings.Split(string(src), "\nfunc ") {
|
||||
if strings.Contains(fn, idiom) && !strings.HasPrefix(fn, "(b *Builder) armRetake(") {
|
||||
holders = append(holders, path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if len(holders) != 0 {
|
||||
t.Errorf("%v waits for a context's deadline and acts: the retake backoff in Builder.armRetake is the one allowed bounded wait (issue 1748); any other wait is a timer", holders)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,15 +103,25 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
return []stdio.Tool{
|
||||
{Name: check.TryTool,
|
||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
|
||||
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"},
|
||||
Description: "Try a delivery group, or one pushed head, before its pull requests are opened (try a group, group try, try several repositories): " +
|
||||
"name the members, each a repository and the exact commit asked (never the branch's head now, so a push after the ask changes nothing of what is tried), " +
|
||||
"or one repository and its head for a try of one member (the seat verb try of mesh-delivery-checker takes members from the registry milestone on; until it is delivered, ask a group through this tool). The checker judges it by the code a group's pull-request check runs, so the two cannot drift: " +
|
||||
"each member merged onto its trunk's head, the unit and the integration level of every member through the build modules, the manifest validator over " +
|
||||
"the members' manifests, the module graph, and the composition on every placed node. Every failure is named (each member, level, test, manifest and node), not " +
|
||||
"the first. A try owes the forge nothing (no status, no Approve asked) and writes no verdict of a pull request; what its passing levels prove is kept, so a pull request's " +
|
||||
"check of the same heads on the same trunk runs only what the try did not pass. A try narrowed to a level or to some tests judges only those, no manifest and no composition, and keeps nothing of tests. " +
|
||||
"Answers the try's id at once; " + check.TryReadTool + " reads where it stands, " + check.TryEndTool + " ends it by hand. A try of the same commits that waits or runs is the answer to a second ask " +
|
||||
"(whatever the order of the members); a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide. A repository the checker checks no level of (one whose trunk the gate does not name) " +
|
||||
"is refused. A try whose ask no build node took within the step bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"members": map[string]any{"type": "array", "description": "the group: one repository at most once, each at the exact commit asked and the trunk commit it is merged onto",
|
||||
"items": map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"}}}},
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>, for a try of one member (instead of members)"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits (with repository)"},
|
||||
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent (with repository)"},
|
||||
"tests": map[string]any{"type": "string", "description": "a go test -run pattern: only those tests run, and nothing is kept"},
|
||||
"level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}, "description": "only this level; both when absent"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
@@ -126,16 +136,32 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
if tr.Now != nil {
|
||||
now = tr.Now
|
||||
}
|
||||
var members []check.TryAskMember
|
||||
if raw, ok := args["members"]; ok {
|
||||
list, isList := raw.([]any)
|
||||
if !isList {
|
||||
return nil, fmt.Errorf("members is a list of {repository, head, trunk}")
|
||||
}
|
||||
for _, item := range list {
|
||||
m, isMap := item.(map[string]any)
|
||||
if !isMap {
|
||||
return nil, fmt.Errorf("a member is {repository, head, trunk}")
|
||||
}
|
||||
members = append(members, check.TryAskMember{Repository: str(m, "repository"), Head: str(m, "head"), Trunk: str(m, "trunk")})
|
||||
}
|
||||
}
|
||||
id, err := check.AskTry(tr.Tries, trunks, check.TryAsk{Repository: str(args, "repository"), Head: str(args, "head"),
|
||||
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level")}, now())
|
||||
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level"), Members: members}, now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"id": id}, nil
|
||||
}},
|
||||
{Name: check.TryReadTool,
|
||||
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
|
||||
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
|
||||
Description: "Where a try (of a group or of one head) stands, read whole, as stored: queued, running, or its end (pass, fail, or error when it could not run, or " +
|
||||
"when its manifests and composition could not be judged), with what was said of it, every member's commit and the trunk commit it was merged onto, the time it took, " +
|
||||
"every failure one line each (members, levels, tests, manifests, the composition per node), the levels' tests run and kept, the composition on every placed node, " +
|
||||
"the notes of the check (manifests judged, module graph, kept verdicts); while it runs, each ask it put as soon as it is put, " +
|
||||
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
|
||||
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
|
||||
@@ -146,7 +172,7 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
return check.ReadTry(tr.Tries, str(args, "id"))
|
||||
}},
|
||||
{Name: check.TryEndTool,
|
||||
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||
Description: "End a try of a group (or of one head) that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
|
||||
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
|
||||
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
|
||||
"takes it within the step bound. A try that ended already is refused.",
|
||||
@@ -170,6 +196,24 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
}
|
||||
}
|
||||
|
||||
// OpenTriesTools is the tool that lists the tries that wait or run, each with its members at their commits, and whether it is
|
||||
// a whole try (the indexer writes the candidate of a whole try's group): the read a part uses at its start to find the tries
|
||||
// whose start event it missed, instead of reading the checker's state.
|
||||
func OpenTriesTools(tr Trying) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: check.TriesTool,
|
||||
Description: "The tries that wait or run now (a try of a group or of one head; try.started and try.ended on the checker's stream say the same), each with its id, state, its members at their exact commits and " +
|
||||
"trunk commits, and whether it is a whole try (not narrowed to a level or to tests, so it is composed and its manifests judged). For a part that " +
|
||||
"missed the checker's try.started or try.ended event (the indexer at its start); checker_try_read reads one try whole. Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
if tr.Tries == nil {
|
||||
return nil, fmt.Errorf("the tries state cannot be read here")
|
||||
}
|
||||
open, err := check.OpenTries(tr.Tries)
|
||||
return map[string]any{"tries": open}, err
|
||||
}}}
|
||||
}
|
||||
|
||||
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
|
||||
const Seat = "mesh-delivery-checker"
|
||||
|
||||
@@ -325,7 +369,7 @@ func main() {
|
||||
_, err := verdicts.Update(key, json.RawMessage(value), revision)
|
||||
return err
|
||||
}}
|
||||
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
|
||||
tools := append(append(append(append(Tools(r), TryTools(trying)...), OpenTriesTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
|
||||
if err := stdio.Serve("", logged(tools)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -8,6 +8,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
@@ -106,6 +108,39 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
|
||||
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
|
||||
t.Error("the verb tried a repository nothing checks")
|
||||
}
|
||||
// A group try (issue 1921): the members, each an exact commit; members and one repository together are refused; the
|
||||
// description says what it does in the words a search for "try a group" finds.
|
||||
later := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(2, 5) },
|
||||
Trunks: func() (map[string]string, error) {
|
||||
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
|
||||
}})
|
||||
g, err := later[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}}})
|
||||
if err != nil || g.(map[string]any)["id"] == id {
|
||||
t.Errorf("a group try = %v, %v, want a try of its own", g, err)
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}},
|
||||
"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40)}); err == nil {
|
||||
t.Error("members and a repository were both taken")
|
||||
}
|
||||
if _, err := tools[0].Run(map[string]any{"members": "novox/mesh-delivery"}); err == nil {
|
||||
t.Error("members that are no list were taken")
|
||||
}
|
||||
for _, tl := range tools {
|
||||
for _, words := range []string{"group"} {
|
||||
if !strings.Contains(strings.ToLower(tl.Description), words) {
|
||||
t.Errorf("%s says nothing of a %s: %q", tl.Name, words, tl.Description)
|
||||
}
|
||||
}
|
||||
}
|
||||
if d := strings.ToLower(tools[0].Description); !strings.Contains(d, "try a group") || !strings.Contains(d, "exact commit") || !strings.Contains(d, "every failure") ||
|
||||
!strings.Contains(d, "composition on every placed node") || !strings.Contains(d, "owes the forge nothing") || !strings.Contains(d, "kept") {
|
||||
t.Errorf("the try's description does not say what a group try does: %q", d)
|
||||
}
|
||||
if open := OpenTriesTools(Trying{Tries: store}); len(open) != 1 || open[0].Name != check.TriesTool || !strings.Contains(open[0].Description, "try.started") {
|
||||
t.Errorf("open tries tool = %+v", open)
|
||||
} else if out, err := open[0].Run(nil); err != nil || len(out.(map[string]any)["tries"].([]tryevent.Open)) != 2 {
|
||||
t.Errorf("open tries = %v, %v, want the two tries asked", out, err)
|
||||
}
|
||||
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
|
||||
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
|
||||
if err != nil || ended.(map[string]any)["state"] != "error" {
|
||||
|
||||
@@ -862,9 +862,11 @@ func (s triesStore) Create(key string, value []byte) error {
|
||||
|
||||
// TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed: a try written as checker_try writes it is taken by the running checker,
|
||||
// judged through the build modules as a pull request's head merged onto its trunk, and ended pass in its key, with no
|
||||
// verdict and no status owed (ADR 0501 §5).
|
||||
// verdict and no status owed (ADR 0501 §5). A whole try waits for the indexer's candidate of its group, which it reads on the
|
||||
// key's watch, and writes the merge previews the indexer makes it from (issue 1921).
|
||||
func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
|
||||
b := startBed(t)
|
||||
asked := b.engine(t, "n1") // a whole try is composed on every placed node, as a group's check is
|
||||
b.start(t, nil)
|
||||
store := triesStore{b.buckets[check.TriesBucket], b.ctx}
|
||||
id, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head1}, time.Now())
|
||||
@@ -872,19 +874,36 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var last check.Try
|
||||
// The indexer's candidate of the try's group (here written by the test), at the heads of its members.
|
||||
raw0, _ := store.Get(id)
|
||||
first, _ := check.DecodeTry(raw0)
|
||||
cand := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: check.TryCase(id, first, nil).HeadsHash,
|
||||
Pulls: []delivery.CandidatePull{{Head: delivery.Head{Repository: repo, Head: head1}, Modules: []string{}}}}
|
||||
cand.FPSet = delivery.FingerprintSet(cand.Pulls)
|
||||
data, err := delivery.EncodeCandidate(cand)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.CandidatesKey(id)
|
||||
if _, err := b.buckets[delivery.CandidatesBucket].Create(b.ctx, ckey, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
raw, _ := store.Get(id)
|
||||
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
|
||||
break
|
||||
}
|
||||
}
|
||||
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" {
|
||||
t.Fatalf("try = %+v, want passed on the trunk", last)
|
||||
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" || last.Composition == nil {
|
||||
t.Fatalf("try = %+v, want passed on the trunk with its composition made", last)
|
||||
}
|
||||
if got := asked(); len(got) != 1 || !strings.HasPrefix(got[0].Candidate, id+".") {
|
||||
t.Errorf("the placed node's engine was asked %+v, want the try's candidate %s.<heads>", got, id)
|
||||
}
|
||||
if b.fake.count("mesh-build-go.build_go_fetch") == 0 {
|
||||
t.Errorf("the try fetched nothing")
|
||||
}
|
||||
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket, delivery.PreviewsBucket} {
|
||||
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket} {
|
||||
if keys, _ := b.buckets[name].Keys(b.ctx); len(keys) != 0 {
|
||||
t.Errorf("a try wrote %s: %v", name, keys)
|
||||
}
|
||||
|
||||
@@ -29,7 +29,10 @@ type judgeDeps struct {
|
||||
trunks *check.Trunks
|
||||
kept check.KV
|
||||
previews check.KV
|
||||
repos *repoModules
|
||||
// candidates answers the indexer's candidate of a try's group (try_candidates.go); nil leaves a whole try with no
|
||||
// manifest or composition judged, said.
|
||||
candidates check.TryCandidates
|
||||
repos *repoModules
|
||||
// published is the indexer's packages state as it is followed (packages.go).
|
||||
published *publishedPackages
|
||||
forge check.Forge
|
||||
@@ -157,7 +160,7 @@ func newJudge(d judgeDeps) check.Judge {
|
||||
nodes = d.nodes
|
||||
}
|
||||
ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf}
|
||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf}
|
||||
if d.nodes != nil && d.builds != nil && d.repos != nil {
|
||||
ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -189,7 +190,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
Verdicts: verdicts.all,
|
||||
}
|
||||
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf,
|
||||
// A whole try's start and end are said on the stream checks, for the indexer to write and delete the candidate of its group.
|
||||
Say: func(ctx context.Context, event string, body tryevent.Event) error {
|
||||
data, _ := json.Marshal(body)
|
||||
_, err := js.Publish(ctx, eventSubject(event), data, jetstream.WithRetryAttempts(10), jetstream.WithRetryWait(time.Second))
|
||||
return err
|
||||
}}
|
||||
announce := func(ctx context.Context, event, key string) {
|
||||
body, _ := json.Marshal(map[string]string{"key": key})
|
||||
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
||||
@@ -202,6 +209,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// The indexer's packages state (ADR 0593 §2): followed once the bus has it, so a Go check builds every mesh package at
|
||||
// its published version; until then a check builds each at the version its go.mod names.
|
||||
published := &publishedPackages{}
|
||||
compositions.Published = published
|
||||
process.AwaitBucket(ctx, js, conn, delivery.PackagesBucket, "a Go check builds each mesh package at the version its go.mod names",
|
||||
logger.Printf).Ready(func(kv jetstream.KeyValue) {
|
||||
w, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { published.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
@@ -216,7 +224,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
|
||||
ch.Judge = newJudge(judgeDeps{
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, candidates: tryCandidates{buckets[delivery.CandidatesBucket]},
|
||||
repos: repos, published: published, builds: builds, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||
announce: announce, compositions: compositions,
|
||||
nodes: openNodeNames(ctx, js, logger.Printf),
|
||||
@@ -333,6 +341,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
})
|
||||
}()
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if check.IsTryGroup(e.Key()) {
|
||||
return // a group try's candidate: its try judges it, and the forge is owed nothing (issue 1921)
|
||||
}
|
||||
if opOf(e) {
|
||||
ch.Candidate(ctx, e.Key(), e.Value())
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// candidateWaitsOnPreview is the None2 of a module the indexer wrote from a pull message alone, before it had the merge
|
||||
// preview of the head (the indexer's NoneNoPreview): its fingerprint comes with the preview, so a candidate that still says
|
||||
// it is not yet the candidate of the heads previewed. It moves to mesh-sdk's delivery package with the next change there.
|
||||
const candidateWaitsOnPreview = "no merge preview is computed"
|
||||
|
||||
// tryCandidates answers the indexer's candidate of a try's group (novox/hq ADR 0593 §7, issue 1921): the candidate the
|
||||
// engines compose and the manifest validator judges, which the indexer makes of the try's previews. It waits for it on the
|
||||
// key's own watch, with no timer between reads, and gives up at the bound.
|
||||
type tryCandidates struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (t tryCandidates) TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error) {
|
||||
key, err := delivery.CandidatesKey(group)
|
||||
if err != nil {
|
||||
return delivery.Candidate{}, err
|
||||
}
|
||||
if bound <= 0 {
|
||||
bound = time.Minute
|
||||
}
|
||||
waiting, cancel := context.WithTimeout(ctx, bound)
|
||||
defer cancel()
|
||||
w, err := t.kv.Watch(waiting, key) // the value now, then each change
|
||||
if err != nil {
|
||||
return delivery.Candidate{}, fmt.Errorf("the indexer's candidate %s cannot be watched: %w", key, err)
|
||||
}
|
||||
defer func() { _ = w.Stop() }()
|
||||
last := "the indexer holds none"
|
||||
for {
|
||||
select {
|
||||
case <-waiting.Done():
|
||||
return delivery.Candidate{}, fmt.Errorf("the indexer wrote no candidate of %s at heads %s within %s (%s)", group, heads, bound, last)
|
||||
case e := <-w.Updates():
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(e.Value())
|
||||
switch {
|
||||
case err != nil:
|
||||
last = "what it holds is not a candidate: " + err.Error()
|
||||
case c.HeadsHash != heads:
|
||||
last = "it holds heads " + c.HeadsHash
|
||||
case waitsOnPreview(c):
|
||||
last = "it has not yet the merge previews' fingerprints"
|
||||
default:
|
||||
return c, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// waitsOnPreview says a module of the candidate has no fingerprint yet because its preview was not read.
|
||||
func waitsOnPreview(c delivery.Candidate) bool {
|
||||
for _, p := range c.Pulls {
|
||||
for _, m := range p.Fps {
|
||||
if m.None2 == candidateWaitsOnPreview {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg=
|
||||
git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -29,24 +29,30 @@ const (
|
||||
nodeEngineModule = "mesh-host"
|
||||
// networkingRepository holds route-proxy, whose manifest the node-engine's proofs of a composition read.
|
||||
networkingRepository = "novox/mesh-networking"
|
||||
// contractsRepository and deliveryRepository hold the seat definitions mesh-sdk's bootstrapped seats are compared with: its
|
||||
// tests read both beside it (beside.Dir "mesh-contracts" and "mesh-delivery"), at the group's head for a member, else at the
|
||||
// trunk (a group try lays them out the same way: besideFor serves the check and the try).
|
||||
contractsRepository = "novox/mesh-contracts"
|
||||
deliveryRepository = DeliveryRepository
|
||||
)
|
||||
|
||||
// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside
|
||||
// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog
|
||||
// checks/deps). Which commit of each is besideCommit's rule. No repository's tests read the controller, which ended by its
|
||||
// unassignment (novox/hq ADR 0590, issue 1126), and its repository is emptied and archived (ADR 0593 §11), so it has no
|
||||
// siblings placed either. The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
|
||||
// siblings placed either. mesh-sdk reads mesh-contracts and mesh-delivery (its bootstrapped seats against their contracts files). The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
|
||||
// §7): mesh-build-go builds it for a compose ask. The node-engine's read mesh-networking since route-proxy's proofs of a
|
||||
// composition moved into them (ADR 0593).
|
||||
var besideRules = map[string][]string{
|
||||
nodeEngineRepo: {catalogueRepository, SDKRepository, networkingRepository},
|
||||
SDKRepository: {catalogueRepository, nodeEngineRepo},
|
||||
SDKRepository: {catalogueRepository, nodeEngineRepo, contractsRepository, deliveryRepository},
|
||||
catalogueRepository: {SDKRepository, nodeEngineRepo},
|
||||
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
|
||||
}
|
||||
|
||||
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the node-engine at the commit the mesh runs
|
||||
// (the commit of the indexer's module mesh-host, its trunk head when none is known); any other sibling, the catalogue
|
||||
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the node-engine at the head of the group's pull
|
||||
// request for it when the group holds one, else at the commit the mesh runs (the commit of the indexer's module mesh-host,
|
||||
// its trunk head when none is known); any other sibling, the catalogue
|
||||
// included, at the head of the group's pull request for it when the group holds one, else at its trunk, since a group's
|
||||
// check judges each member against the group's heads (ADR 0482 §2, ADR 0573 §4; issue 1126: a judge a group changed was
|
||||
// judged against the catalogue's trunk). And first, the change's own repository at the trunk it is merged onto, so a test
|
||||
@@ -63,6 +69,12 @@ func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef,
|
||||
}
|
||||
for _, repo := range besideRules[p.Repository] {
|
||||
switch {
|
||||
case repo == nodeEngineRepo && c.member(repo) != nil:
|
||||
// A group's check judges each member against the group's heads (ADR 0573 §4, ADR 0482 §2), and the group's
|
||||
// node-engine is what runs after its merge: it is placed at the member's head, as any member is. Neither atTrunk
|
||||
// nor runs, so no move of the trunk or of the running commit puts the verdict out of date (besideAtTrunk).
|
||||
member := c.member(repo)
|
||||
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
|
||||
case repo == nodeEngineRepo:
|
||||
commit := ""
|
||||
if j.Runs != nil {
|
||||
|
||||
@@ -55,6 +55,8 @@ type Case struct {
|
||||
Watch func(askID string, step *delivery.Step)
|
||||
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
||||
Progress *Progress
|
||||
// Wait says what a whole try waits for now ("" when nothing); the try keeps it for try-read. Nil for a pull request's check.
|
||||
Wait func(why string)
|
||||
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
|
||||
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
|
||||
Kind string
|
||||
|
||||
@@ -52,7 +52,12 @@ type EngineCompositions struct {
|
||||
// Manifests are the indexer's trunk manifest digests by module, and Verdicts its refusals of them: the catalogue the
|
||||
// sandbox composes with, before the case's own manifests are laid over.
|
||||
Manifests TrunkManifests
|
||||
Verdicts func() map[string]delivery.ManifestVerdict
|
||||
// Published names the latest published version of every mesh package (Examiner.Published): the composer is built
|
||||
// against them and the group's heads, as a check is (issue 1921); nil names the group's heads alone.
|
||||
Published interface {
|
||||
LatestPackages() delivery.PackageVersions
|
||||
}
|
||||
Verdicts func() map[string]delivery.ManifestVerdict
|
||||
}
|
||||
|
||||
// composerHead is the node-engine member's head of a case, or empty.
|
||||
@@ -220,7 +225,7 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
job := delivery.ComposeJob{Fetch: map[string]any{"repository": nodeEngineRepo, "commit": head}, Dir: delivery.RootPath,
|
||||
job := delivery.ComposeJob{Fetch: withPackages(map[string]any{"repository": nodeEngineRepo, "commit": head}, packagesFor(e.Published, c, nodeEngineRepo)), Dir: delivery.RootPath,
|
||||
Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
|
||||
verdict, err := e.Sandbox.Compose(ctx, job)
|
||||
if err != nil {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -150,7 +151,7 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
Machines: map[string]composer.Machine{node: {Composes: false, Reason: "the new composer refuses it"}}}}}
|
||||
}}
|
||||
ec := EngineCompositions{Asker: e, Placed: func() ([]string, error) { return []string{"anchor"}, nil }, Bound: time.Second,
|
||||
Sandbox: sb, Manifests: digests{"web": "sha256:" + strings.Repeat("c", 64)}}
|
||||
Sandbox: sb, Manifests: digests{"web": "sha256:" + strings.Repeat("c", 64)}, Published: published{}}
|
||||
v, err := ec.Compose(context.Background(), host)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -165,6 +166,10 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
|
||||
string(req.Input) != string(withInput.Input) || req.Cases[0].Manifests["web"].Digest == "" {
|
||||
t.Errorf("job = %+v, request %+v", job, req)
|
||||
}
|
||||
// Issue 1921: the composer is built at the group's heads, also a member that publishes no package yet.
|
||||
if pkgs, _ := buildstep.PackagesArg(job.Fetch["packages"]); len(pkgs) != 1 || pkgs[delivery.ForgeModulePrefix+"mesh-delivery"] != head1 {
|
||||
t.Errorf("the composer is fetched with the packages %v, want the group's mesh-delivery member at its head %s alone", pkgs, head1)
|
||||
}
|
||||
if v.State != delivery.CompositionFail || v.SDK != head2 || !strings.Contains(v.Words, "the new composer refuses it") {
|
||||
t.Errorf("verdict = %+v", v)
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
|
||||
gobuild "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/composer"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -1035,8 +1036,25 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
|
||||
// mail's tool while mail awaits web's database: both left out, so neither is refused (ADR 0614 §3, §6).
|
||||
cycle string
|
||||
trunkCycle bool
|
||||
// indexer makes web the holder of the seat mesh-catalog, and the trunk's manifest validator answers for it: "held" with no verdict,
|
||||
// "aside" with a field it does not know; ownJudge has the group change the judge, so the manifest is also judged by the
|
||||
// trunk's, in a second ask (ADR 0633 clause 6, 1585/T22, T23)
|
||||
indexer string
|
||||
ownJudge bool
|
||||
// try judges the same change as a whole group try, whose manifests come from the indexer's candidate of the try's
|
||||
// group (issue 1921): the same judge, the same verdicts.
|
||||
try bool
|
||||
}{
|
||||
{name: "accepted"},
|
||||
{name: "accepted in a group try", try: true},
|
||||
{name: "refused in a group try", try: true, refused: "web has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web has no id`},
|
||||
{name: "a cycle the group try makes", try: true, cycle: "provisions", fails: "the change's module graph has the cycle mail -> web -> mail"},
|
||||
{name: "the indexer accepted by the trunk's manifest validator", indexer: "ok"},
|
||||
{name: "the indexer held by the trunk's manifest validator", indexer: "held", fails: "the indexer web, the holder of the seat mesh-catalog, would never run: the trunk's manifest validator at mesh-delivery's trunk " + strings.Repeat("d", 40) + " does not accept its manifest: the manifest validator holds it, with no verdict yet"},
|
||||
{name: "the indexer with a field the trunk's manifest validator does not know", indexer: "aside", fails: "the manifest validator does not know its fields new_field, so it waits"},
|
||||
{name: "the indexer refused by the trunk's manifest validator", indexer: "refused", refused: "web has no id", fails: "does not accept its manifest: web has no id"},
|
||||
{name: "the indexer under a changed judge is judged by the trunk's as well", indexer: "aside", ownJudge: true, fails: "the manifest validator does not know its fields new_field, so it waits"},
|
||||
{name: "a touched manifest the judge holds", indexer: "heldplain", fails: "holds the manifest of web: it gave no verdict"},
|
||||
{name: "an untouched manifest's problem the trunk had", untouched: "mail says no credential form", trunkMail: "mail says no credential form"},
|
||||
{name: "a problem the change causes in an untouched manifest", untouched: "mail reads a state web no longer keeps",
|
||||
fails: "refuses the manifest of mail: mail reads a state web no longer keeps"},
|
||||
@@ -1077,6 +1095,10 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
|
||||
}
|
||||
b.ex.Read = fixedManifests{strings.Repeat("a", 64): mail, strings.Repeat("b", 64): trunkWeb, web: changed}
|
||||
}
|
||||
if row.indexer != "" && row.indexer != "heldplain" {
|
||||
b.ex.Read = fixedManifests{strings.Repeat("a", 64): {Module: "mail"}, strings.Repeat("b", 64): {Module: "web"},
|
||||
web: {Module: "web", Claims: []catalogue.Claim{{Name: CatalogSeat}}}}
|
||||
}
|
||||
var judged []delivery.ComposeJob
|
||||
answer := b.asks.Answer
|
||||
b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||
@@ -1085,6 +1107,12 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
|
||||
}
|
||||
judged = append(judged, *a.Compose)
|
||||
read := composer.ManifestRead{Accepted: row.refused == "", Refused: row.refused}
|
||||
switch row.indexer {
|
||||
case "held", "heldplain":
|
||||
read = composer.ManifestRead{}
|
||||
case "aside":
|
||||
read.SetAside = "new_field"
|
||||
}
|
||||
verdict, _ := json.Marshal(composer.Verdict{V: composer.Version, SDK: "exe:1", Cases: []composer.CaseVerdict{{Name: JudgeCase,
|
||||
Manifests: map[string]composer.ManifestRead{"web": read, "mail": {Accepted: row.untouched == "", Refused: row.untouched}}},
|
||||
{Name: JudgeTrunk, Manifests: map[string]composer.ManifestRead{"web": {Accepted: true}, "mail": {Accepted: row.trunkMail == "", Refused: row.trunkMail}}}}})
|
||||
@@ -1092,9 +1120,53 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
|
||||
return delivery.Step{ID: a.ID, State: delivery.StepPassed, Result: result}
|
||||
}
|
||||
p := pullOf(repoA, 7, head1)
|
||||
if row.ownJudge {
|
||||
p = pullOf(DeliveryRepository, 7, head1)
|
||||
b.fetch.Changed = []string{JudgeDir + "/judge/judge.go"}
|
||||
}
|
||||
p.Modules = []string{"web"}
|
||||
p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}}
|
||||
v := b.judge(t, caseOf(t, p)).Verdict
|
||||
// Issue 1921: the judge is built at the group's heads too, also a member that publishes no package yet.
|
||||
cs := caseOf(t, p, pullOf("novox/mesh-networking", 9, head2))
|
||||
if row.try {
|
||||
cs = TryCase("try-1", Try{Members: []TryMember{{Repository: repoA, Head: head1, Trunk: trunkA}}}, nil)
|
||||
b.ex.Candidates = &fakeTryCandidates{cand: delivery.Candidate{FPSet: "fps", Pulls: []delivery.CandidatePull{p}}}
|
||||
}
|
||||
v := b.judge(t, cs).Verdict
|
||||
if row.indexer == "ok" && !slices.ContainsFunc(cs.notes, func(n string) bool {
|
||||
return strings.Contains(n, "the new indexer's manifest of web is accepted by the trunk's manifest validator")
|
||||
}) {
|
||||
t.Errorf("%s: the case says %v, want the new indexer's manifest said accepted by the trunk's manifest validator", row.name, cs.notes)
|
||||
}
|
||||
// Issue 1921: the judge of the change is built at the group's heads (also a member that publishes no package yet), but
|
||||
// the trunk's manifest validator that rules on a new indexer is built at the latest published versions alone, never
|
||||
// at a group's head (ADR 0633 clause 6): a group with a member of its own then judges the indexer a second time.
|
||||
trunkRuns := 0
|
||||
if row.indexer != "" && row.indexer != "heldplain" && (row.ownJudge || !row.try) {
|
||||
trunkRuns = 1
|
||||
}
|
||||
if want := 1 + trunkRuns; len(judged) != want {
|
||||
t.Fatalf("%s: %d judge asks %+v, want %d", row.name, len(judged), judged, want)
|
||||
}
|
||||
for i, j := range judged {
|
||||
pkgs, _ := buildstep.PackagesArg(j.Fetch["packages"])
|
||||
atGroup := pkgs[delivery.ForgeModulePrefix+"mesh-networking"] == head2
|
||||
switch trunkRun := trunkRuns == 1 && i == len(judged)-1; {
|
||||
case row.try:
|
||||
case trunkRun && atGroup:
|
||||
t.Errorf("%s: the trunk's manifest validator is built with the group's head: %v", row.name, pkgs)
|
||||
case !trunkRun && !atGroup:
|
||||
t.Errorf("%s: the judge at %v is fetched with the packages %v, want the group's mesh-networking member at its head %s", row.name, j.Fetch["commit"], pkgs, head2)
|
||||
}
|
||||
}
|
||||
if row.ownJudge && (len(judged) != 2 || judged[0].Fetch["commit"] != head1 || judged[1].Fetch["commit"] != trunk) {
|
||||
t.Fatalf("%s: judged %+v, want the head's judge and then the trunk's", row.name, judged)
|
||||
}
|
||||
if row.ownJudge {
|
||||
judged = judged[1:]
|
||||
} else if trunkRuns == 1 {
|
||||
judged = judged[1:] // the group's run; the trunk's is the one read below
|
||||
}
|
||||
if len(judged) != 1 || judged[0].Fetch["repository"] != DeliveryRepository || judged[0].Fetch["commit"] != trunk ||
|
||||
judged[0].Package != JudgePackage || judged[0].Dir != JudgeDir {
|
||||
t.Fatalf("%s: judged %+v (verdict %+v)", row.name, judged, v)
|
||||
|
||||
@@ -30,7 +30,7 @@ type ManifestReader interface {
|
||||
// refuseCycle fails the check for a cycle of the change's module graph, naming it; nil when the case touches no manifest,
|
||||
// when nothing reads manifests, or when there is none. A manifest that cannot be read fails the check, said, never passes it.
|
||||
func (st *state) refuseCycle(ctx context.Context) *delivery.Verdict {
|
||||
if st.j.Manifests == nil || st.j.Read == nil || st.c.Try != nil {
|
||||
if st.j.Manifests == nil || st.j.Read == nil || st.narrowTry() {
|
||||
return nil
|
||||
}
|
||||
trunk := st.j.Manifests.ManifestDigests()
|
||||
|
||||
@@ -139,8 +139,18 @@ type Examiner struct {
|
||||
Nodes interface {
|
||||
Names(ctx context.Context) ([]string, error)
|
||||
}
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
// Candidates answers the indexer's candidate of a try's group once it wrote it from the try's previews, the manifests
|
||||
// and fingerprints a group try's manifest validator and composition judge (issue 1921); nil leaves a whole try with no
|
||||
// manifest or composition judged, said.
|
||||
Candidates TryCandidates
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
}
|
||||
|
||||
// TryCandidates answers the candidate the indexer holds for a try's group at its heads hash, waiting for it within the
|
||||
// bound; an error says it did not come (and why).
|
||||
type TryCandidates interface {
|
||||
TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error)
|
||||
}
|
||||
|
||||
// CompositionReader makes a check's composition verdict; an error is only a case that cannot be composed at all.
|
||||
@@ -246,6 +256,50 @@ type state struct {
|
||||
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
|
||||
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
|
||||
unpreviewed []string
|
||||
// noCandidate says why a whole try has no candidate of its group from the indexer, so its manifests were not judged and
|
||||
// it was not composed; empty when it has one, and for a pull request's check.
|
||||
noCandidate string
|
||||
}
|
||||
|
||||
// narrowTry says the case is a try narrowed to a level or to some tests: it judges no manifest, graph or composition.
|
||||
func (st *state) narrowTry() bool { return st.c.Try != nil && !st.c.Try.Whole() }
|
||||
|
||||
// takeTryCandidate gives a whole try's pulls the fingerprints and manifests the indexer's candidate of its group holds
|
||||
// (the group's pull-request check has them from its candidate, CaseOf), once the indexer wrote it from the try's previews.
|
||||
// A try with no such candidate is said so, in a note and in its composition, and is no pass (Tryer.run).
|
||||
func (st *state) takeTryCandidate(ctx context.Context) {
|
||||
if st.c.Try == nil || !st.c.Try.Whole() {
|
||||
return
|
||||
}
|
||||
no := func(why string) {
|
||||
st.noCandidate = "the group's manifests and composition were not judged: " + why
|
||||
st.c.note(st.noCandidate)
|
||||
}
|
||||
if st.j.Candidates == nil {
|
||||
no("the checker reads no candidate of the indexer's")
|
||||
return
|
||||
}
|
||||
if len(st.unpreviewed) > 0 {
|
||||
no("no preview of a merge of " + strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer)")
|
||||
return
|
||||
}
|
||||
if st.c.Wait != nil {
|
||||
st.c.Wait("the indexer's candidate of the group at its heads (up to " + st.j.buildWait().String() + ")")
|
||||
defer st.c.Wait("")
|
||||
}
|
||||
cand, err := st.j.Candidates.TryCandidate(ctx, st.c.Group, st.c.HeadsHash, st.j.buildWait())
|
||||
if err != nil {
|
||||
no(err.Error())
|
||||
return
|
||||
}
|
||||
for i, p := range st.c.Pulls {
|
||||
for _, cp := range cand.Pulls {
|
||||
if cp.Repository == p.Repository && cp.Head.Head == p.Head {
|
||||
st.c.Pulls[i].Modules, st.c.Pulls[i].Fps = cp.Modules, cp.Fps
|
||||
}
|
||||
}
|
||||
}
|
||||
st.c.FPSet = cand.FPSet
|
||||
}
|
||||
|
||||
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
|
||||
@@ -256,6 +310,7 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
||||
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
|
||||
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
|
||||
tests := st.judgeAll(ctx)
|
||||
st.takeTryCandidate(ctx)
|
||||
if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil {
|
||||
return st.finish(ctx, *v), nil
|
||||
}
|
||||
@@ -282,7 +337,14 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
|
||||
return
|
||||
}
|
||||
if st.j.Compositions == nil || st.c.Try != nil {
|
||||
if st.narrowTry() {
|
||||
return
|
||||
}
|
||||
if st.noCandidate != "" {
|
||||
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: st.noCandidate}
|
||||
return
|
||||
}
|
||||
if st.j.Compositions == nil {
|
||||
return
|
||||
}
|
||||
if len(st.unpreviewed) > 0 {
|
||||
@@ -433,7 +495,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||
// Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests
|
||||
// failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0).
|
||||
switch {
|
||||
case st.c.Try != nil: // a try owes the indexer no preview: no pull request holds its head
|
||||
case st.narrowTry(): // a try of a level or some tests owes the indexer no preview: it judges no manifest
|
||||
case len(merged) == len(modules):
|
||||
st.j.writePreview(ctx, p, merged)
|
||||
default:
|
||||
@@ -562,7 +624,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
|
||||
return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch))
|
||||
}
|
||||
if len(t.fetched.Conflicts) > 0 {
|
||||
if st.c.Try == nil {
|
||||
if !st.narrowTry() {
|
||||
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
||||
}
|
||||
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
|
||||
|
||||
@@ -97,7 +97,7 @@ func judgeRequest(trunk map[string]string, pulls []Pull) (req composer.Request,
|
||||
// refuses. A case that touches no manifest, or an examiner that reads no trunk manifests, judges nothing; a judge that
|
||||
// cannot be asked or does not answer fails the check, said, never passes it.
|
||||
func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
|
||||
if st.j.Manifests == nil || st.c.Try != nil {
|
||||
if st.j.Manifests == nil || st.narrowTry() {
|
||||
return nil
|
||||
}
|
||||
req, touched := judgeRequest(st.j.Manifests.ManifestDigests(), st.c.Pulls)
|
||||
@@ -120,26 +120,12 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
|
||||
req.Predefined = list
|
||||
st.c.note("the manifests are judged against the predefined keys of " + from)
|
||||
}
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return fail(goModule, err.Error())
|
||||
cv, trunkCase, v := st.runJudge(ctx, req, touched, commit, at, st.packagesOf(DeliveryRepository))
|
||||
if v != nil {
|
||||
return v
|
||||
}
|
||||
job := delivery.ComposeJob{Fetch: map[string]any{"repository": DeliveryRepository, "commit": commit}, Dir: JudgeDir,
|
||||
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
|
||||
At: st.j.now().UTC(), Compose: &job}
|
||||
st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "")
|
||||
st.c.Asked(a.ID)
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error())
|
||||
}
|
||||
step, err := st.j.Asks.Wait(ctx, a)
|
||||
if err != nil {
|
||||
return fail(goModule, "the judge of the change's manifests: "+err.Error())
|
||||
}
|
||||
cv, trunkCase, why := judgeAnswer(step)
|
||||
if why != "" {
|
||||
return fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why))
|
||||
if v := st.refuseUnmovableIndexer(ctx, req, touched, cv, at); v != nil {
|
||||
return v
|
||||
}
|
||||
// A new or changed manifest is judged in full; of one the change leaves as it is, a problem the trunk already had is a
|
||||
// warning (it is not refused for a rule it already broke), and a problem the change causes in it, which the trunk case
|
||||
@@ -147,7 +133,13 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
|
||||
var refused, warned []string
|
||||
for _, module := range slices.Sorted(maps.Keys(cv.Manifests)) {
|
||||
r := cv.Manifests[module]
|
||||
if r.Accepted || r.Refused == "" {
|
||||
if r.Accepted {
|
||||
continue
|
||||
}
|
||||
if r.Refused == "" { // held: a verdict not given is not one given (ADR 0633 clause 6)
|
||||
if slices.Contains(touched, module) {
|
||||
refused = append(refused, fmt.Sprintf("the indexer's judge at %s holds the manifest of %s: it gave no verdict", at, module))
|
||||
}
|
||||
continue
|
||||
}
|
||||
had := map[string]bool{}
|
||||
@@ -183,6 +175,100 @@ func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
|
||||
return nil
|
||||
}
|
||||
|
||||
// runJudge asks the Go build seat to build the judge at the commit and run it over the request, and reads its change and
|
||||
// trunk cases; a failed verdict when it cannot be asked or does not answer.
|
||||
func (st *state) runJudge(ctx context.Context, req composer.Request, touched []string, commit, at string, versions delivery.PackageVersions) (change, trunk composer.CaseVerdict, v *delivery.Verdict) {
|
||||
raw, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return change, trunk, fail(goModule, err.Error())
|
||||
}
|
||||
job := delivery.ComposeJob{Fetch: withPackages(map[string]any{"repository": DeliveryRepository, "commit": commit}, versions), Dir: JudgeDir,
|
||||
Package: JudgePackage, Request: raw, Reader: "the indexer's judge at " + at}
|
||||
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskCheck, Language: delivery.AskGo, Asker: delivery.CheckerModule,
|
||||
At: st.j.now().UTC(), Compose: &job}
|
||||
st.c.Progress.Set(fmt.Sprintf("the judge ask %s of the manifests of %s", a.ID, strings.Join(touched, ", ")), "")
|
||||
st.c.Asked(a.ID)
|
||||
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||
return change, trunk, fail(goModule, "the judge of the change's manifests could not be asked: "+err.Error())
|
||||
}
|
||||
step, err := st.j.Asks.Wait(ctx, a)
|
||||
if err != nil {
|
||||
return change, trunk, fail(goModule, "the judge of the change's manifests: "+err.Error())
|
||||
}
|
||||
change, trunk, why := judgeAnswer(step)
|
||||
if why != "" {
|
||||
return change, trunk, fail(goModule, fmt.Sprintf("the indexer's judge at %s did not judge the change's manifests: %s", at, why))
|
||||
}
|
||||
return change, trunk, nil
|
||||
}
|
||||
|
||||
// CatalogSeat is the seat the indexer holds: the module whose manifest claims it is the indexer, found by the claim and
|
||||
// never by its name (novox/hq ADR 0633 clause 6, ADR 0601 §4).
|
||||
const CatalogSeat = "mesh-catalog"
|
||||
|
||||
// refuseUnmovableIndexer fails the check when the trunk's manifest validator, the one the running parts hold, would
|
||||
// refuse or hold the change's new indexer manifest: such an indexer would never run, and every revision behind it would
|
||||
// wait for ever (ADR 0633 clause 6). The indexer is the touched module whose manifest claims the seat mesh-catalog.
|
||||
// When the change judges with its own head's manifest validator (it changes it), the manifest is judged by the trunk's
|
||||
// as well. The verdicts read are the trunk's manifest validator's: its change case, where refused, held (not accepted
|
||||
// with no reason) or with a field set aside (a field the running manifest validator does not know waits, ADR 0601 §3)
|
||||
// each fail it, naming the module and the problems.
|
||||
func (st *state) refuseUnmovableIndexer(ctx context.Context, req composer.Request, touched []string, cv composer.CaseVerdict, judgedAt string) *delivery.Verdict {
|
||||
if st.j.Read == nil {
|
||||
return nil
|
||||
}
|
||||
var indexer string
|
||||
for _, module := range touched {
|
||||
m, err := st.j.Read.Manifest(ctx, req.Cases[0].Manifests[module].Digest)
|
||||
if err != nil {
|
||||
return fail(goModule, fmt.Sprintf("the manifest of %s cannot be read, so the indexer cannot be ruled out: %v", module, err))
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
if c.Name == CatalogSeat {
|
||||
indexer = module
|
||||
}
|
||||
}
|
||||
}
|
||||
if indexer == "" {
|
||||
return nil
|
||||
}
|
||||
// The validator the running parts hold is the trunk's, built at the latest published versions alone, never at the group's
|
||||
// heads (ADR 0633 clause 6): a group's mesh-sdk or mesh-contracts head must not change the validator that judges the new
|
||||
// indexer. The head's run is reused when it was that validator already.
|
||||
if trunkVersions := st.trunkJudgeVersions(); st.judgeHead != "" || !maps.Equal(st.packagesOf(DeliveryRepository), trunkVersions) {
|
||||
trunk := ""
|
||||
if st.j.Trunks != nil {
|
||||
trunk = st.j.Trunks.Heads()[DeliveryRepository]
|
||||
}
|
||||
if trunk == "" {
|
||||
return fail(goModule, "the new indexer cannot be judged by the trunk's manifest validator: the trunk head of "+DeliveryRepository+" is not known")
|
||||
}
|
||||
var v *delivery.Verdict
|
||||
judgedAt = "mesh-delivery's trunk " + trunk
|
||||
if cv, _, v = st.runJudge(ctx, req, touched, trunk, judgedAt, trunkVersions); v != nil {
|
||||
return v
|
||||
}
|
||||
}
|
||||
r, ok := cv.Manifests[indexer]
|
||||
var problems []string
|
||||
switch {
|
||||
case !ok:
|
||||
problems = append(problems, "the manifest validator gave no verdict")
|
||||
case r.Refused != "":
|
||||
problems = append(problems, strings.Split(r.Refused, "\n")...)
|
||||
case !r.Accepted:
|
||||
problems = append(problems, "the manifest validator holds it, with no verdict yet")
|
||||
case r.SetAside != "":
|
||||
problems = append(problems, "the manifest validator does not know its fields "+r.SetAside+", so it waits")
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
return fail(goModule, fmt.Sprintf("the indexer %s, the holder of the seat %s, would never run: the trunk's manifest validator at %s does not accept its manifest: %s",
|
||||
indexer, CatalogSeat, judgedAt, strings.Join(problems, "; ")))
|
||||
}
|
||||
st.c.note(fmt.Sprintf("the new indexer's manifest of %s is accepted by the trunk's manifest validator at %s", indexer, judgedAt))
|
||||
return nil
|
||||
}
|
||||
|
||||
// judgeAnswer reads the judge's change and trunk cases from its step, or why it cannot.
|
||||
func judgeAnswer(step delivery.Step) (change, trunk composer.CaseVerdict, why string) {
|
||||
if step.State != delivery.StepPassed {
|
||||
@@ -257,3 +343,9 @@ func (st *state) judgedByMember(p Pull, merged []Fetched) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// trunkJudgeVersions are the versions the trunk's manifest validator is built at: the latest published version of each
|
||||
// package, with no group member's head (ADR 0633 clause 6).
|
||||
func (st *state) trunkJudgeVersions() delivery.PackageVersions {
|
||||
return packagesFor(st.j.Published, &Case{}, DeliveryRepository)
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/grouppackages"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -31,37 +31,28 @@ type Forge interface {
|
||||
File(ctx context.Context, repository, path, ref string) (string, error)
|
||||
}
|
||||
|
||||
// packagesOf is the version of each mesh package a Go check of repository builds against (ADR 0573 §2, §4): the latest
|
||||
// published version of each package the mesh's modules use (Published), and for a package one of the case's pull
|
||||
// requests publishes, that pull request's head (mesh-sdk's Go module whenever mesh-sdk is a member). A package of
|
||||
// repository itself comes with the tree the check fetched, and is left out. Nil for none.
|
||||
// packagesOf is the version of each mesh package a Go build of repository (a check ask, the judge, the composer) builds
|
||||
// against (ADR 0573 §2, §4): grouppackages.Heads over the latest published version of each package the mesh's modules use
|
||||
// (Published) and the heads of the case's pull requests, so a member that publishes no package yet is named too (issue
|
||||
// 1921). Nil for none.
|
||||
func (st *state) packagesOf(repository string) delivery.PackageVersions {
|
||||
versions := delivery.PackageVersions{}
|
||||
if st.j.Published != nil {
|
||||
maps.Copy(versions, st.j.Published.LatestPackages())
|
||||
return packagesFor(st.j.Published, st.c, repository)
|
||||
}
|
||||
|
||||
// packagesFor is packagesOf for a case and the published versions: every producer of a build at a group's heads takes its
|
||||
// versions from here, so none leaves a member out.
|
||||
func packagesFor(published interface {
|
||||
LatestPackages() delivery.PackageVersions
|
||||
}, c *Case, repository string) delivery.PackageVersions {
|
||||
var latest delivery.PackageVersions
|
||||
if published != nil {
|
||||
latest = published.LatestPackages()
|
||||
}
|
||||
if st.c.member(SDKRepository) != nil {
|
||||
if _, named := versions[SDKModulePath]; !named {
|
||||
versions[SDKModulePath] = ""
|
||||
}
|
||||
heads := map[string]string{}
|
||||
for _, m := range c.Pulls {
|
||||
heads[m.Repository] = m.Head
|
||||
}
|
||||
for pkg := range versions {
|
||||
repo, _, ok := delivery.PackageRepository(pkg)
|
||||
if !ok || delivery.SameRepository(repo, repository) {
|
||||
delete(versions, pkg)
|
||||
continue
|
||||
}
|
||||
if m := st.c.member(repo); m != nil && len(m.Head) == 40 {
|
||||
versions[pkg] = m.Head
|
||||
}
|
||||
if versions[pkg] == "" {
|
||||
delete(versions, pkg)
|
||||
}
|
||||
}
|
||||
if len(versions) == 0 {
|
||||
return nil
|
||||
}
|
||||
return versions
|
||||
return grouppackages.Heads(latest, heads, repository)
|
||||
}
|
||||
|
||||
// unheardWords ends the words of a failed read of the forge that reached nobody: the forge was not heard (UnheardWords).
|
||||
@@ -80,3 +71,17 @@ func forgeMachine(err error, node string) string {
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
// withPackages puts the versions into a fetch's arguments (the fetch tool's packages argument, which a compose ask has no
|
||||
// field for: a compose ask names none, StepAsk.Packages), so the composer or the judge is built at the group's heads.
|
||||
func withPackages(fetch map[string]any, versions delivery.PackageVersions) map[string]any {
|
||||
if len(versions) == 0 {
|
||||
return fetch
|
||||
}
|
||||
packages := make(map[string]any, len(versions))
|
||||
for pkg, version := range versions {
|
||||
packages[pkg] = version
|
||||
}
|
||||
fetch["packages"] = packages
|
||||
return fetch
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/grouppackages"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -60,6 +61,48 @@ func TestAGroupMemberIsBuiltAgainstTheSDKMembersHeadWhateverItsGoModNames(t *tes
|
||||
if got := packagesOfFetch(t, b, "novox/mesh-catalog"); len(got) != 2 || got[SDKModulePath] != trunkS || got[contract] != trunkA {
|
||||
t.Errorf("a pull request alone is built against %v, want every package at its latest published version", got)
|
||||
}
|
||||
// A member that publishes no package yet is still named, by its root module path, at its head (issue 1921).
|
||||
b = newBench(t)
|
||||
b.ex.Published = published{}
|
||||
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf("novox/mesh-host", 8, head2))).Verdict; v.State != delivery.VerdictPass {
|
||||
t.Fatalf("an unpublished member: verdict = %+v", v)
|
||||
}
|
||||
if got := packagesOfFetch(t, b, repoA); len(got) != 1 || got["git.novox.be/novox/mesh-host"] != head2 {
|
||||
t.Errorf("the member's packages = %v, want the unpublished member's root at its head %s alone", got, head2)
|
||||
}
|
||||
// The function every producer takes the versions from (issue 1921): rows for what a build at a group's heads is asked.
|
||||
h := func(c string) string { return strings.Repeat(c, 40) }
|
||||
const (
|
||||
contracts = "git.novox.be/novox/mesh-contracts/go/clients/assignerclient"
|
||||
tools = "git.novox.be/novox/mesh-tools/go/x"
|
||||
)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
latest delivery.PackageVersions
|
||||
heads map[string]string
|
||||
own string
|
||||
want delivery.PackageVersions
|
||||
}{
|
||||
{"none", nil, nil, "novox/mesh-host", nil},
|
||||
{"trunk latest alone", delivery.PackageVersions{tools: h("a")}, nil, "novox/mesh-host", delivery.PackageVersions{tools: h("a")}},
|
||||
{"a member at its head replaces the latest", delivery.PackageVersions{SDKModulePath: h("a")}, map[string]string{SDKRepository: h("b")}, "novox/mesh-host",
|
||||
delivery.PackageVersions{SDKModulePath: h("b")}},
|
||||
{"a member that publishes no package yet is named by its root module", delivery.PackageVersions{SDKModulePath: h("a")},
|
||||
map[string]string{"novox/mesh-contracts": h("c")}, "novox/mesh-host",
|
||||
delivery.PackageVersions{SDKModulePath: h("a"), "git.novox.be/novox/mesh-contracts": h("c")}},
|
||||
{"mesh-sdk is named by its go module before any package of it is known", nil, map[string]string{SDKRepository: h("b")}, "novox/mesh-host",
|
||||
delivery.PackageVersions{SDKModulePath: h("b")}},
|
||||
{"a member whose packages are known gets no root", delivery.PackageVersions{contracts: h("a")},
|
||||
map[string]string{"novox/mesh-contracts": h("c")}, "novox/mesh-host", delivery.PackageVersions{contracts: h("c")}},
|
||||
{"the build's own repository is left out", delivery.PackageVersions{contracts: h("a")},
|
||||
map[string]string{"novox/mesh-contracts": h("c")}, "novox/mesh-contracts", nil},
|
||||
{"a head that is no full commit names nothing", nil, map[string]string{"novox/mesh-contracts": "abc"}, "novox/mesh-host", nil},
|
||||
} {
|
||||
got := grouppackages.Heads(tc.latest, tc.heads, tc.own)
|
||||
if !maps.Equal(got, tc.want) {
|
||||
t.Errorf("%s: got %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type runsAt map[string]string
|
||||
@@ -111,10 +154,16 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
||||
}
|
||||
// Alone, mesh-sdk's tests read the catalogue's trunk, and the verdict names it.
|
||||
b = newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2})
|
||||
const contracts = "novox/mesh-contracts"
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2, contracts: trunkA, DeliveryRepository: trunkA2})
|
||||
if out := b.judge(t, caseOf(t, pullOf(SDKRepository, 7, head1))); out.Beside[catalogue] != trunkA {
|
||||
t.Errorf("the siblings named for the verdict alone = %v, want the catalogue at its trunk %s", out.Beside, trunkA)
|
||||
}
|
||||
// mesh-sdk's tests compare its bootstrapped seats with their contracts files, read beside it: mesh-contracts and
|
||||
// mesh-delivery, at their trunks alone (a group try is placed by the same besideFor, so it lays them out the same way).
|
||||
if got := besideOf(t, b, SDKRepository); got[contracts].Commit != trunkA || got[DeliveryRepository].Commit != trunkA2 {
|
||||
t.Errorf("beside mesh-sdk alone = %+v, want mesh-contracts at %s and mesh-delivery at %s", got, trunkA, trunkA2)
|
||||
}
|
||||
if c := besideOf(t, b, SDKRepository)[catalogue]; c.Commit != trunkA || c.Ref != "" {
|
||||
t.Errorf("the catalogue beside mesh-sdk alone = %+v, want its trunk %s", c, trunkA)
|
||||
}
|
||||
@@ -168,6 +217,18 @@ func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
|
||||
if h := got[nodeEngineRepo]; h.Commit != hostRuns {
|
||||
t.Errorf("the node-engine beside a catalogue change = %+v, want the commit the mesh runs %s", h, hostRuns)
|
||||
}
|
||||
// A group that holds the node-engine is judged against its head, which is what runs after its merge, never the commit
|
||||
// the mesh runs now; the verdict names it in no carry (it is a member, not read at a trunk or a running commit).
|
||||
b = newBench(t)
|
||||
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, nodeEngineRepo: trunkA2})
|
||||
b.ex.Runs = runsAt{nodeEngineModule: hostRuns}
|
||||
out = b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1), pullOf(nodeEngineRepo, 8, head2)))
|
||||
if h := besideOf(t, b, catalogueRepository)[nodeEngineRepo]; h.Commit != head2 || h.Ref != "refs/pull/8/head" {
|
||||
t.Errorf("the node-engine beside a catalogue change in a group holding it = %+v, want the member's head %s", h, head2)
|
||||
}
|
||||
if out.Beside[nodeEngineRepo] != "" {
|
||||
t.Errorf("the verdict names the member node-engine in its carry: %v", out.Beside)
|
||||
}
|
||||
}
|
||||
|
||||
// 1887/A12: a mesh-lab head gets a check: with its trunk known it is judged like any repository, its Go fetch finding
|
||||
|
||||
@@ -164,6 +164,7 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
||||
if err != nil {
|
||||
return VerdictRead{}, err
|
||||
}
|
||||
keys = slices.DeleteFunc(keys, IsTryGroup)
|
||||
sort.Strings(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
|
||||
@@ -444,6 +445,7 @@ func (r Reader) Status() (Status, error) {
|
||||
if err != nil {
|
||||
return st, err
|
||||
}
|
||||
keys = slices.DeleteFunc(keys, IsTryGroup) // a try's candidate is judged by its try, never as a check
|
||||
sort.Strings(keys)
|
||||
total := len(keys)
|
||||
keys, cut := capKeys(keys, "")
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -27,6 +28,8 @@ const (
|
||||
TryTool = "checker_try"
|
||||
TryReadTool = "checker_try_read"
|
||||
TryEndTool = "checker_try_end"
|
||||
// TriesTool lists the tries that wait or run (OpenTries).
|
||||
TriesTool = "checker_tries"
|
||||
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
|
||||
TriesState = "tries"
|
||||
TriesBucket = delivery.CheckerModule + "_" + TriesState
|
||||
@@ -62,18 +65,28 @@ var (
|
||||
|
||||
// Try is the value of a tries key.
|
||||
type Try struct {
|
||||
V int `json:"v"`
|
||||
ID string `json:"id"`
|
||||
V int `json:"v"`
|
||||
ID string `json:"id"`
|
||||
// Repository, Head, Trunk and Branch are the first member's (the try's headline, and all a try of one repository
|
||||
// names); Members are every member of the try, by repository: each an exact commit merged onto a trunk commit (a
|
||||
// group try, issue 1921). A try written before members read as one member.
|
||||
Repository string `json:"repository"`
|
||||
// Head is the commit tried; Trunk the trunk commit it is merged onto, and Branch the trunk's branch when it was asked by name.
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Members []TryMember `json:"members,omitempty"`
|
||||
// Tests is a go test -run pattern: only those tests run, and nothing is kept. Level is unit or integration: only that
|
||||
// level runs. Empty is every test of both levels.
|
||||
Tests string `json:"tests,omitempty"`
|
||||
Level string `json:"level,omitempty"`
|
||||
State string `json:"state"`
|
||||
// Told is true from the moment a whole try's start is said on the checker's stream until its end is said there: kept in the
|
||||
// try's own state, so an end that is written and not said (the checker stopped between) is said when the checker starts.
|
||||
Told bool `json:"told,omitempty"`
|
||||
// Waiting is what the running try waits for now (its group's candidate from the indexer, ADR 0639 §5), empty when it waits for nothing;
|
||||
// try-read gives it, so a try that sits still says why.
|
||||
Waiting string `json:"waiting,omitempty"`
|
||||
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
|
||||
Said string `json:"said,omitempty"`
|
||||
// Took is the time from its start to its end, as Go writes a duration.
|
||||
@@ -83,6 +96,11 @@ type Try struct {
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
|
||||
Machines []string `json:"machines,omitempty"`
|
||||
// Composition is what the composition on every placed node came to (a whole try, not one narrowed to a level or to
|
||||
// some tests), and Notes the words the check noted on its way (the manifests judged, the module graph, the kept
|
||||
// verdicts taken): the same as a group's pull-request check reports.
|
||||
Composition *delivery.Composition `json:"composition,omitempty"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
|
||||
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
|
||||
Asks []string `json:"asks,omitempty"`
|
||||
@@ -94,6 +112,23 @@ type Try struct {
|
||||
Ended time.Time `json:"ended,omitzero"`
|
||||
}
|
||||
|
||||
// TryMember is one repository of a try: the exact commit asked, never the branch's head now, and the trunk commit it is
|
||||
// merged onto. Branch is the trunk's branch when the trunk was asked by name.
|
||||
type TryMember struct {
|
||||
Repository string `json:"repository"`
|
||||
Head string `json:"head"`
|
||||
Trunk string `json:"trunk"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
}
|
||||
|
||||
// MembersOf are the try's members: its Members, or the one its headline names.
|
||||
func (t Try) MembersOf() []TryMember {
|
||||
if len(t.Members) > 0 {
|
||||
return t.Members
|
||||
}
|
||||
return []TryMember{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk, Branch: t.Branch}}
|
||||
}
|
||||
|
||||
// TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream.
|
||||
const TryAsked = "asked"
|
||||
|
||||
@@ -148,6 +183,17 @@ func (w *tryWriter) heard(askID string, step *delivery.Step) {
|
||||
}
|
||||
}
|
||||
|
||||
// waiting writes what the try waits for now ("" when it waits for nothing).
|
||||
func (w *tryWriter) waiting(why string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopped || w.t.Waiting == why {
|
||||
return
|
||||
}
|
||||
w.t.Waiting = why
|
||||
w.write()
|
||||
}
|
||||
|
||||
// write writes the record at the revision written last; a refusal stops the writes.
|
||||
func (w *tryWriter) write() {
|
||||
data, err := encodeTry(w.t)
|
||||
@@ -195,6 +241,7 @@ func DecodeTry(raw []byte) (Try, error) {
|
||||
if !tryKeyOK.MatchString(t.ID) || t.Repository == "" || t.Head == "" || t.Trunk == "" {
|
||||
return Try{}, errors.New("not a try: it names no id, repository, head or trunk")
|
||||
}
|
||||
t.Members = t.MembersOf()
|
||||
return t, nil
|
||||
}
|
||||
|
||||
@@ -206,6 +253,14 @@ func encodeTry(t Try) ([]byte, error) {
|
||||
// TryAsk is what checker_try is given.
|
||||
type TryAsk struct {
|
||||
Repository, Head, Trunk, Tests, Level string
|
||||
// Members are the repositories of a group try, each an exact commit and, optionally, its trunk (a commit, or the
|
||||
// trunk's branch for its head now); the single Repository, Head and Trunk above are one member. Both at once are refused.
|
||||
Members []TryAskMember
|
||||
}
|
||||
|
||||
// TryAskMember is one repository of a group try as it is asked.
|
||||
type TryAskMember struct {
|
||||
Repository, Head, Trunk string
|
||||
}
|
||||
|
||||
// TryStore is the tries state as the tools see it.
|
||||
@@ -221,21 +276,16 @@ func NotChecked(repository string) error {
|
||||
return fmt.Errorf("no level of %s is checked by the checker", repository)
|
||||
}
|
||||
|
||||
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
|
||||
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the
|
||||
// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
|
||||
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
|
||||
// tryMember checks one member as it is asked and resolves its trunk: the gate's trunk head of its repository, a commit
|
||||
// asked, or the trunk's branch (its head now). A repository with no trunk head is no repository the checker checks.
|
||||
func tryMember(trunks map[string]string, a TryAskMember) (TryMember, error) {
|
||||
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
|
||||
head := strings.ToLower(strings.TrimSpace(a.Head))
|
||||
switch {
|
||||
case !tryRepoOK.MatchString(repo) || strings.Contains(repo, ".."):
|
||||
return "", fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
|
||||
return TryMember{}, fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
|
||||
case !tryHeadOK.MatchString(head):
|
||||
return "", fmt.Errorf("the head %q is not a full commit, 40 or 64 hex digits", a.Head)
|
||||
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
|
||||
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
|
||||
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
|
||||
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
|
||||
return TryMember{}, fmt.Errorf("the head %q of %s is not a full commit, 40 or 64 hex digits", a.Head, repo)
|
||||
}
|
||||
trunkHead := ""
|
||||
for r, h := range trunks {
|
||||
@@ -244,20 +294,67 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
||||
}
|
||||
}
|
||||
if trunkHead == "" {
|
||||
return "", NotChecked(repo)
|
||||
return TryMember{}, NotChecked(repo)
|
||||
}
|
||||
t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
|
||||
m := TryMember{Repository: repo, Head: head}
|
||||
switch trunk := strings.TrimSpace(a.Trunk); {
|
||||
case trunk == "":
|
||||
t.Trunk = trunkHead
|
||||
m.Trunk = trunkHead
|
||||
case tryHeadOK.MatchString(strings.ToLower(trunk)):
|
||||
t.Trunk = strings.ToLower(trunk)
|
||||
m.Trunk = strings.ToLower(trunk)
|
||||
case tryTrunkOK.MatchString(trunk) && !strings.Contains(trunk, ".."):
|
||||
// The checker judges pull requests into the trunk alone (ADR 0484 clause 11): a branch named is the trunk's head now.
|
||||
t.Trunk, t.Branch = trunkHead, trunk
|
||||
m.Trunk, m.Branch = trunkHead, trunk
|
||||
default:
|
||||
return "", fmt.Errorf("the trunk %q is neither a full commit nor a branch", a.Trunk)
|
||||
return TryMember{}, fmt.Errorf("the trunk %q of %s is neither a full commit nor a branch", a.Trunk, repo)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// sameMembers says whether two tries ask the same commits onto the same trunk commits.
|
||||
func sameMembers(a, b []TryMember) bool {
|
||||
return slices.EqualFunc(a, b, func(x, y TryMember) bool {
|
||||
return x.Repository == y.Repository && x.Head == y.Head && x.Trunk == y.Trunk
|
||||
})
|
||||
}
|
||||
|
||||
// AskTry writes a new try, queued, and answers its id: a try of the same commits that waits or runs is the answer to a second
|
||||
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. A try is a group of
|
||||
// one or more members (issue 1921), each an exact commit, never the branch's head now: a push after the ask changes nothing
|
||||
// of what is tried. trunks are the gate's trunk heads of every repository (source.main): a repository with none is no
|
||||
// repository the checker checks. A trunk asked by name is its trunk head now.
|
||||
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
|
||||
asked := a.Members
|
||||
if a.Repository != "" || a.Head != "" || a.Trunk != "" {
|
||||
if len(asked) > 0 {
|
||||
return "", errors.New("name the members, or one repository with its head, not both")
|
||||
}
|
||||
asked = []TryAskMember{{Repository: a.Repository, Head: a.Head, Trunk: a.Trunk}}
|
||||
}
|
||||
if len(asked) == 0 {
|
||||
return "", errors.New("name what to try: members (repository and head each), or a repository and its head")
|
||||
}
|
||||
switch {
|
||||
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
|
||||
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
|
||||
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
|
||||
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
|
||||
}
|
||||
var members []TryMember
|
||||
for _, m := range asked {
|
||||
tm, err := tryMember(trunks, m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if slices.ContainsFunc(members, func(o TryMember) bool { return o.Repository == tm.Repository }) {
|
||||
return "", fmt.Errorf("%s is named twice: a try holds one commit of a repository", tm.Repository)
|
||||
}
|
||||
members = append(members, tm)
|
||||
}
|
||||
slices.SortFunc(members, func(x, y TryMember) int { return strings.Compare(x.Repository, y.Repository) })
|
||||
first := members[0]
|
||||
t := Try{Repository: first.Repository, Head: first.Head, Trunk: first.Trunk, Branch: first.Branch, Members: members,
|
||||
Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
|
||||
keys, err := store.Keys()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("the tries cannot be listed: %w", err)
|
||||
@@ -291,10 +388,10 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
||||
if !live[i] {
|
||||
continue
|
||||
}
|
||||
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
|
||||
if sameMembers(cur.MembersOf(), t.Members) && cur.Tests == t.Tests && cur.Level == t.Level {
|
||||
return cur.ID, nil
|
||||
}
|
||||
said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)
|
||||
said := fmt.Sprintf("%s (%s, %s)", cur.ID, sayMembers(cur.MembersOf()), cur.State)
|
||||
if cur.HoldsANode() {
|
||||
holding = append(holding, said)
|
||||
} else {
|
||||
@@ -379,12 +476,50 @@ func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// IsTryGroup says a candidates key is the group of a try (the indexer makes the candidate of a group try's previews, for the
|
||||
// manifest validator and the composition): the try judges it, so the checker's check of candidates leaves it alone, writes
|
||||
// no verdict and owes the forge no status for it.
|
||||
func IsTryGroup(key string) bool { return tryKeyOK.MatchString(key) }
|
||||
|
||||
// TryOptions narrow what a try runs; a pull request's check has none.
|
||||
type TryOptions struct {
|
||||
Level string
|
||||
Run string
|
||||
}
|
||||
|
||||
// Whole says the try runs everything a group's pull-request check runs (both levels of every member, the manifests judged by
|
||||
// the manifest validator, the module graph, the composition on every placed node): it is narrowed to neither a level nor
|
||||
// some tests. Such a try judges by the same code as the check, so the two cannot drift (issue 1921).
|
||||
func (o *TryOptions) Whole() bool { return o != nil && o.Level == "" && o.Run == "" }
|
||||
|
||||
// TryCase is the case a try is judged as: one pull of the group per member at its exact commit, with no pull request number
|
||||
// (no ref: the commit is fetched as it is), the group named by the try's key, and the heads hash its candidate is keyed by.
|
||||
func TryCase(key string, t Try, watch func(string, *delivery.Step)) *Case {
|
||||
cs := &Case{Key: key, Group: key, Watch: watch, Trunk: map[string]string{}, Queued: t.Asked, Progress: &Progress{},
|
||||
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
|
||||
var heads []delivery.Head
|
||||
for _, m := range t.MembersOf() {
|
||||
cs.Pulls = append(cs.Pulls, Pull{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
|
||||
cs.Trunk[m.Repository] = m.Trunk
|
||||
heads = append(heads, delivery.Head{Repository: m.Repository, Head: m.Head})
|
||||
}
|
||||
cs.HeadsHash, _ = delivery.HeadsHash(heads)
|
||||
cs.Heads = cs.HeadsHash
|
||||
return cs
|
||||
}
|
||||
|
||||
// compositionMade says whether a composition was made; notMadeWords what it says when it was not.
|
||||
func compositionMade(c *delivery.Composition) bool {
|
||||
return c != nil && c.State != delivery.CompositionNotRun
|
||||
}
|
||||
|
||||
func notMadeWords(c *delivery.Composition) string {
|
||||
if c == nil || c.Words == "" {
|
||||
return "no reason was given"
|
||||
}
|
||||
return c.Words
|
||||
}
|
||||
|
||||
// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check (a check ask, ADR
|
||||
// 0525 §1), and writes its end. It holds no timer: a try starts when its key is written.
|
||||
type Tryer struct {
|
||||
@@ -396,7 +531,13 @@ type Tryer struct {
|
||||
Now func() time.Time
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
// Say publishes an event of the checker's stream (tryevent.Started, tryevent.Ended) with its body; nil says nothing. The
|
||||
// indexer, which hears them, writes and deletes the candidate of a whole try's group.
|
||||
// It answers the publish's error, nil only when the stream acknowledged it: an end not acknowledged is said again.
|
||||
Say func(ctx context.Context, event string, body tryevent.Event) error
|
||||
|
||||
mu sync.Mutex
|
||||
ending map[string]bool // tries whose end is being said now
|
||||
wg sync.WaitGroup
|
||||
inflight map[string]context.CancelFunc
|
||||
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
|
||||
@@ -436,6 +577,7 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
|
||||
stop()
|
||||
}
|
||||
tr.mu.Unlock()
|
||||
tr.sendEnd(ctx, key) // ended by hand while it ran, or its end was written and not said: the indexer drops its candidate
|
||||
return
|
||||
}
|
||||
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
|
||||
@@ -491,6 +633,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
return
|
||||
}
|
||||
t.State, t.Started = TryRunning, tr.now().UTC()
|
||||
t.Told = (&TryOptions{Level: t.Level, Run: t.Tests}).Whole() // its start is said below; its end is owed from now on
|
||||
data, err := encodeTry(t)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -502,12 +645,15 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
return
|
||||
}
|
||||
progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev}
|
||||
cs := &Case{Key: key, Group: key, Watch: progress.heard,
|
||||
Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}},
|
||||
Trunk: map[string]string{t.Repository: t.Trunk},
|
||||
Queued: t.Asked,
|
||||
Progress: &Progress{},
|
||||
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
|
||||
if t.Told {
|
||||
started := tryevent.Event{ID: key}
|
||||
for _, m := range t.MembersOf() {
|
||||
started.Members = append(started.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
tr.say(ctx, tryevent.Started, started)
|
||||
}
|
||||
cs := TryCase(key, t, progress.heard)
|
||||
cs.Wait = progress.waiting
|
||||
judging := ctx
|
||||
if tr.NotTaken > 0 {
|
||||
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
|
||||
@@ -517,12 +663,18 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
|
||||
}
|
||||
t, rev = progress.end()
|
||||
t.Waiting = ""
|
||||
v := out.Verdict
|
||||
t.Ended = tr.now().UTC()
|
||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
|
||||
t.Composition = v.Composition
|
||||
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
||||
for _, n := range cs.Notes() {
|
||||
if len(cs.Pulls) > 1 {
|
||||
t.Said = sayMembers(t.MembersOf()) + ": " + t.Said
|
||||
}
|
||||
t.Notes = cs.Notes()
|
||||
for _, n := range t.Notes {
|
||||
t.Said += " | " + n
|
||||
}
|
||||
switch {
|
||||
@@ -532,6 +684,16 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
if IsLostAsk(v) || IsNotTaken(v) {
|
||||
t.Failed = []string{v.Failed.Words}
|
||||
}
|
||||
case v.State == delivery.VerdictPass && cs.Try.Whole() && compositionMade(v.Composition) && strings.Contains(v.Composition.Words, delivery.ComposeNoAnswer):
|
||||
// A node that gave no answer passes a group's check with a warning; a try that is to say the group composes says it
|
||||
// did not, naming the node (ADR 0637 §3).
|
||||
t.State = TryError
|
||||
t.Failed = []string{"the composition was not made on every placed node: " + v.Composition.Words}
|
||||
case v.State == delivery.VerdictPass && cs.Try.Whole() && !compositionMade(v.Composition):
|
||||
// A whole try that judged the tests and not the manifests and the composition has not tried what its group's check
|
||||
// judges: it is no pass, and it says why (failure must be loud). It is an error, not the change's failure.
|
||||
t.State = TryError
|
||||
t.Failed = []string{"the composition was not made: " + notMadeWords(v.Composition)}
|
||||
case v.State == delivery.VerdictPass:
|
||||
t.State = TryPass
|
||||
default:
|
||||
@@ -549,6 +711,97 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
||||
return
|
||||
}
|
||||
tr.logf("%s: %s", key, t.State)
|
||||
tr.sendEnd(ctx, key)
|
||||
}
|
||||
|
||||
// sendEnd says the end of an ended try whose start was said (Told) with the state it ended in, then clears Told in the try's
|
||||
// own state; a checker that stopped between the two says it again at its start. Every end is said so: pass, fail, error and
|
||||
// by hand.
|
||||
func (tr *Tryer) sendEnd(ctx context.Context, key string) {
|
||||
tr.mu.Lock()
|
||||
if tr.ending[key] {
|
||||
tr.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if tr.ending == nil {
|
||||
tr.ending = map[string]bool{}
|
||||
}
|
||||
tr.ending[key] = true
|
||||
tr.mu.Unlock()
|
||||
defer func() { tr.mu.Lock(); delete(tr.ending, key); tr.mu.Unlock() }()
|
||||
raw, rev, found, err := tr.Tries.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
t, err := DecodeTry(raw)
|
||||
if err != nil || !t.Over() || !t.Told {
|
||||
return
|
||||
}
|
||||
if tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) != nil {
|
||||
return // not acknowledged: Told stays, and the next Put or Reconsider says the end again
|
||||
}
|
||||
t.Told = false
|
||||
data, err := encodeTry(t)
|
||||
if err == nil {
|
||||
_, err = tr.Tries.Update(ctx, key, data, rev)
|
||||
}
|
||||
if err != nil && !errors.Is(err, ErrConflict) {
|
||||
tr.logf("%s: its end was said and Told could not be cleared: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// say publishes a try event; a failure is said in the log (the indexer finds the open tries again at its start).
|
||||
func (tr *Tryer) say(ctx context.Context, event string, body tryevent.Event) error {
|
||||
if tr.Say == nil {
|
||||
return nil
|
||||
}
|
||||
err := tr.Say(ctx, event, body)
|
||||
if err != nil {
|
||||
tr.logf("%s of %s could not be said: %v", event, body.ID, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// OpenTries are every try that waits or runs now: what checker_tries answers, so a part that missed an event (the indexer at
|
||||
// its start) finds them without reading the checker's state. All the keys are read (an old open try is as open as a new one), in
|
||||
// parallel.
|
||||
func OpenTries(store TryStore) ([]tryevent.Open, error) {
|
||||
keys, err := store.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the tries cannot be listed: %w", err)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if len(keys[i]) != len(keys[j]) {
|
||||
return len(keys[i]) > len(keys[j])
|
||||
}
|
||||
return keys[i] > keys[j]
|
||||
})
|
||||
found := make([]*tryevent.Open, len(keys))
|
||||
if err := parallel(len(keys), func(i int) error {
|
||||
raw, err := store.Get(keys[i])
|
||||
if err != nil || len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
t, err := DecodeTry(raw)
|
||||
if err != nil || t.Over() {
|
||||
return nil
|
||||
}
|
||||
o := tryevent.Open{ID: t.ID, State: t.State, Whole: (&TryOptions{Level: t.Level, Run: t.Tests}).Whole()}
|
||||
for _, m := range t.MembersOf() {
|
||||
o.Members = append(o.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
|
||||
}
|
||||
found[i] = &o
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []tryevent.Open{} // none open is an empty list, never null: a reader tells it from no answer (issue 1921)
|
||||
for _, o := range found {
|
||||
if o != nil {
|
||||
out = append(out, *o)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A
|
||||
@@ -579,6 +832,15 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O
|
||||
return out, true
|
||||
}
|
||||
|
||||
// sayMembers names a try's members, each repository at its short commit.
|
||||
func sayMembers(ms []TryMember) string {
|
||||
var out []string
|
||||
for _, m := range ms {
|
||||
out = append(out, fmt.Sprintf("%s@%.12s", m.Repository, m.Head))
|
||||
}
|
||||
return strings.Join(out, " + ")
|
||||
}
|
||||
|
||||
// orNone joins the tries said, or says none.
|
||||
func orNone(said []string) string {
|
||||
if len(said) == 0 {
|
||||
|
||||
@@ -3,11 +3,14 @@ package check
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -70,6 +73,25 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR
|
||||
}
|
||||
// Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or
|
||||
// their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts.
|
||||
// A group try is the same try whatever the order of its members; the commits asked are the commits tried.
|
||||
group := TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2}, {Repository: repoA, Head: head1}}}
|
||||
gid, err := AskTry(store, tryTrunks, group, at.Add(time.Second))
|
||||
if err != nil || gid == id {
|
||||
t.Fatalf("the group try = %q, %v, want a try of its own", gid, err)
|
||||
}
|
||||
group.Members[0], group.Members[1] = group.Members[1], group.Members[0]
|
||||
if again, err := AskTry(store, tryTrunks, group, at.Add(time.Minute)); err != nil || again != gid {
|
||||
t.Errorf("the group asked in another order answered %q, %v, want %q", again, err, gid)
|
||||
}
|
||||
if g := readTry(t, kv, gid); len(g.Members) != 2 || g.Members[0].Repository != repoA || g.Members[0].Head != head1 || g.Members[1].Trunk != trunkS {
|
||||
t.Errorf("group try = %+v, want its members by repository at the commits asked", g.Members)
|
||||
}
|
||||
if open, err := OpenTries(store); err != nil || len(open) != 2 || !open[0].Whole || len(open[1].Members) != 2 && len(open[0].Members) != 2 {
|
||||
t.Errorf("open tries = %+v, %v, want the two queued", open, err)
|
||||
}
|
||||
if !IsTryGroup(gid) || IsTryGroup("feat/x") || IsTryGroup("fix/1921-try-1") {
|
||||
t.Errorf("which candidates keys are a try's group: %s is, a branch is not", gid)
|
||||
}
|
||||
onNode := func(i int) Try {
|
||||
return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning,
|
||||
Steps: []TryStep{{Ask: "a", State: delivery.StepRunning}}}
|
||||
@@ -112,59 +134,154 @@ func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agr
|
||||
}
|
||||
}
|
||||
for _, bad := range []TryAsk{{Repository: repoA, Head: "abc"}, {Repository: repoA, Head: head1, Level: "e2e"},
|
||||
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"}} {
|
||||
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"},
|
||||
// A group try (issue 1921): members or one repository, never both or none; one commit per repository; every member checked.
|
||||
{},
|
||||
{Repository: repoA, Head: head1, Members: []TryAskMember{{Repository: repoSDK, Head: head2}}},
|
||||
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoA, Head: head2}}},
|
||||
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoSDK, Head: "main"}}},
|
||||
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: "novox/no-trunk", Head: head2}}}} {
|
||||
if _, err := AskTry(store, tryTrunks, bad, time.Now()); err == nil {
|
||||
t.Errorf("%+v was taken", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5
|
||||
// fakeTryCandidates answers the indexer's candidate of a try's group from memory, recording the keys asked.
|
||||
type fakeTryCandidates struct {
|
||||
cand delivery.Candidate
|
||||
err error
|
||||
asked []string
|
||||
// during is called while the try waits for the candidate.
|
||||
during func()
|
||||
}
|
||||
|
||||
func (f *fakeTryCandidates) TryCandidate(_ context.Context, group, heads string, _ time.Duration) (delivery.Candidate, error) {
|
||||
if f.during != nil {
|
||||
f.during()
|
||||
}
|
||||
f.asked = append(f.asked, group+"."+heads)
|
||||
return f.cand, f.err
|
||||
}
|
||||
|
||||
// wholeBench makes the bench judge a whole try as the checker does: the composition on every placed node and the
|
||||
// indexer's candidate of the try's group.
|
||||
func wholeBench(b *bench, cand delivery.Candidate) (*fakeCompositions, *fakeTryCandidates) {
|
||||
comps := &fakeCompositions{v: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14", "shanks"}, Took: 40}}
|
||||
cands := &fakeTryCandidates{cand: cand}
|
||||
b.ex.Compositions, b.ex.Candidates = comps, cands
|
||||
return comps, cands
|
||||
}
|
||||
|
||||
// A group try (issue 1921): several repositories each at the exact commit asked, judged by the code of a group's pull-request
|
||||
// check; the two cannot drift, and a later check of the same heads keeps what the try passed.
|
||||
func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5, issue 1921
|
||||
ctx := context.Background()
|
||||
kv := newMemKV()
|
||||
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: trunkA}, time.Now())
|
||||
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2, Trunk: trunkS}, {Repository: repoA, Head: head1, Trunk: trunkA}}}, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
comps, cands := wholeBench(b, delivery.Candidate{FPSet: "fps-of-the-try"})
|
||||
written := &writtenTries{memKV: kv}
|
||||
tr := &Tryer{Tries: written, Judge: b.ex}
|
||||
var waiting string
|
||||
cands.during = func() { waiting = readTry(t, kv, id).Waiting } // ADR 0639 §5: the stored try says what it waits for
|
||||
var said []string
|
||||
tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
|
||||
said = append(said, fmt.Sprintf("%s %s %d %s", event, e.ID, len(e.Members), e.State))
|
||||
return nil
|
||||
}}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
// The indexer hears the start with the members at their commits, and the end (it writes and deletes the group's candidate).
|
||||
if len(said) != 2 || said[0] != "try.started "+id+" 2 " || said[1] != "try.ended "+id+" 0 "+TryPass {
|
||||
t.Errorf("events = %v, want the start with 2 members and the end", said)
|
||||
}
|
||||
// What the indexer reads: tryevent.Decode accepts the events as said and refuses a body that is no try's.
|
||||
for _, row := range []struct {
|
||||
name, event, body string
|
||||
ok bool
|
||||
}{
|
||||
{"a start with its members", tryevent.Started, `{"id":"` + id + `","members":[{"repository":"` + repoA + `","head":"` + head1 + `","trunk":"` + trunkA + `"}]}`, true},
|
||||
{"an end with its state", tryevent.Ended, `{"id":"` + id + `","state":"pass"}`, true},
|
||||
{"an id that is no try's", tryevent.Ended, `{"id":"pull-7","state":"pass"}`, false},
|
||||
{"a start with no member", tryevent.Started, `{"id":"` + id + `"}`, false},
|
||||
{"a body that is no JSON", tryevent.Ended, `try-7`, false},
|
||||
} {
|
||||
if e, err := tryevent.Decode(row.event, []byte(row.body)); (err == nil) != row.ok || (row.ok && e.ID != id) {
|
||||
t.Errorf("Decode of %s = %+v, %v, want ok=%v", row.name, e, err, row.ok)
|
||||
}
|
||||
}
|
||||
got := readTry(t, kv, id)
|
||||
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
|
||||
if !strings.Contains(waiting, "candidate") || got.Waiting != "" {
|
||||
t.Errorf("waiting while the candidate was awaited = %q, after the end %q: want the reason, then none", waiting, got.Waiting)
|
||||
}
|
||||
if got.Told { // the end was said: nothing is owed
|
||||
t.Errorf("the ended try still owes its end: %+v", got)
|
||||
}
|
||||
if open, err := OpenTries(memTries{kv}); err != nil || len(open) != 0 {
|
||||
t.Errorf("open tries after the end = %v, %v", open, err)
|
||||
} else if raw, _ := json.Marshal(open); string(raw) != "[]" { // issue 1921: none open is [] on the wire, never null
|
||||
t.Errorf("none open is answered %s, want []", raw)
|
||||
}
|
||||
if got.State != TryPass || got.Took == "" || len(got.Members) != 2 || got.Members[0].Repository != repoA || got.Members[1].Head != head2 ||
|
||||
got.Said == "" || got.Levels[delivery.LevelUnit].Run+got.Levels[delivery.LevelUnit].Kept != 4 { // the bench gives both members the same digests
|
||||
t.Fatalf("try = %+v", got)
|
||||
}
|
||||
// Issue 1890: while it runs, the try names its ask as soon as it is put, and then where its step stands; the end
|
||||
// keeps each ask's node and state.
|
||||
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked ||
|
||||
written.running[2].Steps[0].Node != "shanks" {
|
||||
t.Errorf("the running try's writes = %+v, want its ask named while it ran, then its node", written.running)
|
||||
if got.Composition == nil || got.Composition.State != delivery.CompositionPass || len(got.Machines) == 0 {
|
||||
t.Errorf("the try's composition = %+v, want the composition on every placed node, as a check's", got.Composition)
|
||||
}
|
||||
if len(got.Steps) != 1 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed {
|
||||
t.Errorf("the ended try's steps = %+v", got.Steps)
|
||||
// The same key the indexer's candidate of the group has: the try's id and the heads of its members.
|
||||
if len(comps.asked) != 1 || comps.asked[0] != id || len(cands.asked) != 1 || !strings.HasPrefix(cands.asked[0], id+".") {
|
||||
t.Errorf("composed %v, candidate asked %v, want the try's key %s", comps.asked, cands.asked, id)
|
||||
}
|
||||
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
|
||||
if len(got.Asks) != 1 {
|
||||
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
|
||||
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
|
||||
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
|
||||
// Issue 1890: while it runs, the try names its asks as soon as they are put, and then where each step stands.
|
||||
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked {
|
||||
t.Errorf("the running try's writes = %+v, want its ask named while it ran", written.running)
|
||||
}
|
||||
if len(got.Steps) != 2 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed || len(got.Asks) != 2 {
|
||||
t.Errorf("the ended try's steps = %+v, asks %v", got.Steps, got.Asks)
|
||||
}
|
||||
for _, a := range got.Asks { // the delivery view counts their steps as a try's (build_usage)
|
||||
if _, err := delivery.AskStepKey("shanks", a); err != nil {
|
||||
t.Errorf("the try names %q, which is no ask id: %v", a, err)
|
||||
}
|
||||
}
|
||||
// Each member is fetched at the exact commit asked, merged onto its own trunk, with no pull request's ref.
|
||||
fetch := b.steps.tool("fetch")
|
||||
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
|
||||
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
|
||||
if len(fetch) != 2 {
|
||||
t.Fatalf("fetch = %+v, want one per member", fetch)
|
||||
}
|
||||
if keys, _ := b.prev.Keys(ctx); len(keys) != 0 {
|
||||
t.Errorf("a try wrote the previews %v", keys)
|
||||
for _, f := range fetch {
|
||||
want := map[string][2]string{repoA: {head1, trunkA}, repoSDK: {head2, trunkS}}[f.args["repository"].(string)]
|
||||
if f.args["commit"] != want[0] || f.args["base"] != want[1] || f.args["ref"] != "" {
|
||||
t.Errorf("fetch = %+v, want the exact commit %s merged onto %s", f.args, want[0], want[1])
|
||||
}
|
||||
}
|
||||
// A whole try writes the merge previews the indexer makes its candidate from; it writes no status and asks no Approve.
|
||||
if keys, _ := b.prev.Keys(ctx); len(keys) != 2 {
|
||||
t.Errorf("a whole try wrote the previews %v, want one per member", keys)
|
||||
}
|
||||
// An ended try is not run again.
|
||||
raw, _, _, _ = kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
if n := len(b.steps.tool("fetch")); n != 1 {
|
||||
if n := len(b.steps.tool("fetch")); n != 2 {
|
||||
t.Errorf("an ended try was fetched again: %d fetches", n)
|
||||
}
|
||||
// The kept verdicts: what the try passed is kept under the digests of the merged trees, so the pull-request check of
|
||||
// the same head on the same trunk runs none of it again (and a push of other code runs only what changed).
|
||||
before := len(b.steps.tool("test"))
|
||||
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||
if out.Verdict.State != delivery.VerdictPass || out.Verdict.Levels[delivery.LevelUnit].Kept == 0 || out.Verdict.Levels[delivery.LevelUnit].Run != 0 ||
|
||||
out.Verdict.Levels[delivery.LevelIntegration].Run != 0 {
|
||||
t.Errorf("the check of the tried head = %+v, want everything the try passed kept and nothing run", out.Verdict.Levels)
|
||||
}
|
||||
if after := len(b.steps.tool("test")); after != before {
|
||||
t.Errorf("the check of the tried head ran %d test steps, want none: the try's passes are kept", after-before)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
|
||||
@@ -175,10 +292,16 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex}
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(context.Context, string, tryevent.Event) error {
|
||||
t.Error("a try of one level and some tests said an event: it has no candidate")
|
||||
return nil
|
||||
}}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
if open, _ := OpenTries(memTries{kv}); len(open) != 0 {
|
||||
t.Errorf("open = %v", open)
|
||||
}
|
||||
tests := b.steps.tool("test")
|
||||
if len(tests) != 1 || tests[0].args["level"] != delivery.LevelIntegration || tests[0].args["run"] != "TestBus|TestStore" {
|
||||
t.Fatalf("tests = %+v, want the integration level alone with the run pattern", tests)
|
||||
@@ -192,6 +315,24 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
|
||||
if got := readTry(t, kv, id); got.State != TryPass || !strings.Contains(got.Said, "only the tests TestBus|TestStore") {
|
||||
t.Errorf("try = %+v", got)
|
||||
}
|
||||
t.Run("a try of one level keeps only that level", func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: delivery.LevelUnit}, time.Now())
|
||||
b := newBench(t)
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
keys, _ := b.kept.Keys(ctx)
|
||||
if got := readTry(t, kv, id); got.State != TryPass || len(keys) == 0 {
|
||||
t.Fatalf("try = %+v, kept %v", got, keys)
|
||||
}
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, delivery.LevelUnit+".") {
|
||||
t.Errorf("a try of the unit level kept %s", k)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) {
|
||||
@@ -208,26 +349,135 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T)
|
||||
!strings.Contains(got.Failed[1], "integration: fixture/failing/bad.TestThreeIsFour") { // both levels ran (the operator, 2026-11-02)
|
||||
t.Errorf("try = %+v", got)
|
||||
}
|
||||
// A whole try that could not judge the manifests and the composition is no pass; a failed composition is named beside
|
||||
// the failed tests (the group's check does as much), and a candidate of other heads is not the try's.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
cand *fakeTryCandidates
|
||||
comp delivery.CompositionVerdict
|
||||
state, said string
|
||||
}{
|
||||
{name: "the indexer wrote no candidate", cand: &fakeTryCandidates{err: errors.New("the indexer wrote no candidate within 5m0s")}, state: TryError, said: "the indexer wrote no candidate within 5m0s"},
|
||||
{name: "a node that gave no composition answer", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x", Warnings: []string{"g14: " + delivery.ComposeNoAnswer}},
|
||||
state: TryError, said: "g14: " + delivery.ComposeNoAnswer},
|
||||
{name: "a composition refused on a node", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "not composed on g14: web: needs a state mail does not keep"},
|
||||
state: TryFail, said: "not composed on g14: web: needs a state mail does not keep"},
|
||||
} {
|
||||
kv := newMemKV()
|
||||
b := newBench(t)
|
||||
comps, _ := wholeBench(b, delivery.Candidate{})
|
||||
b.ex.Candidates, comps.v = row.cand, row.comp
|
||||
if row.comp.State == "" {
|
||||
comps.v.State = delivery.CompositionPass
|
||||
}
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
var ends []string
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
|
||||
if event == tryevent.Ended {
|
||||
ends = append(ends, e.State)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
raw, _, _, _ := kv.Get(ctx, id)
|
||||
tr.Put(ctx, id, raw)
|
||||
tr.Wait()
|
||||
if len(ends) != 1 || ends[0] != row.state { // every end says its state: error as well as fail
|
||||
t.Errorf("%s: ends said = %v, want once %s", row.name, ends, row.state)
|
||||
}
|
||||
if got := readTry(t, kv, id); got.State != row.state || !strings.Contains(strings.Join(got.Failed, " | ")+got.Said, row.said) {
|
||||
t.Errorf("%s: try = %s, failed %v, said %q, want %s naming %q", row.name, got.State, got.Failed, got.Said, row.state, row.said)
|
||||
}
|
||||
}
|
||||
// No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4).
|
||||
}
|
||||
|
||||
func TestATryLeftRunningIsTakenOver(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
// Left running by a checker that stopped.
|
||||
raw, rev, _, _ := kv.Get(ctx, id)
|
||||
cur, _ := DecodeTry(raw)
|
||||
cur.State = TryRunning
|
||||
data, _ := encodeTry(cur)
|
||||
if _, err := kv.Update(ctx, id, data, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := newBench(t)
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex}
|
||||
tr.Reconsider(ctx)
|
||||
tr.Wait()
|
||||
if got := readTry(t, kv, id); got.State != TryPass {
|
||||
t.Errorf("the try left running = %+v, want taken over and passed", got)
|
||||
// Whether the start was said is in the try's own state (Told), so a checker that starts later says the end of a try it did
|
||||
// not start, whatever way the try ended.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
told bool
|
||||
failFirst bool // the stream does not acknowledge the first end said
|
||||
after func(t *testing.T, kv *memKV, id string)
|
||||
ends []string
|
||||
}{
|
||||
{name: "the first publish of the end is not acknowledged", told: true, failFirst: true, ends: []string{TryPass}},
|
||||
{name: "left running after its start was said", told: true, ends: []string{TryPass}},
|
||||
{name: "left running with no start said", told: false, ends: []string{TryPass}},
|
||||
{name: "ended by hand after the start was said", told: true, ends: []string{TryError}, after: func(t *testing.T, kv *memKV, id string) {
|
||||
if _, err := EndTry(endable{kv}, id, "stuck", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}},
|
||||
} {
|
||||
t.Run(row.name, func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
// Left running by a checker that stopped.
|
||||
raw, rev, _, _ := kv.Get(ctx, id)
|
||||
cur, _ := DecodeTry(raw)
|
||||
cur.State, cur.Told = TryRunning, row.told
|
||||
data, _ := encodeTry(cur)
|
||||
if _, err := kv.Update(ctx, id, data, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if row.after != nil {
|
||||
row.after(t, kv, id)
|
||||
}
|
||||
b := newBench(t)
|
||||
wholeBench(b, delivery.Candidate{})
|
||||
var ends []string
|
||||
failed := false
|
||||
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
|
||||
if event == tryevent.Ended {
|
||||
if row.failFirst && !failed {
|
||||
failed = true
|
||||
return errors.New("the stream did not acknowledge")
|
||||
}
|
||||
ends = append(ends, e.State)
|
||||
}
|
||||
return nil
|
||||
}}
|
||||
tr.Reconsider(ctx)
|
||||
tr.Wait()
|
||||
if row.failFirst && !readTry(t, kv, id).Told {
|
||||
t.Errorf("an end the stream did not acknowledge cleared Told: it is never said again")
|
||||
}
|
||||
tr.Reconsider(ctx) // said once: the second look finds nothing owed
|
||||
tr.Wait()
|
||||
if row.after == nil {
|
||||
if got := readTry(t, kv, id); got.State != TryPass {
|
||||
t.Errorf("the try left running = %+v, want taken over and passed", got)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(ends, row.ends) || readTry(t, kv, id).Told {
|
||||
t.Errorf("ends said = %v, want %v, and nothing owed after", ends, row.ends)
|
||||
}
|
||||
})
|
||||
}
|
||||
// An ended try is kept TryKeep and removed by the next Reconsider after.
|
||||
t.Run("an ended try is kept seven days", func(t *testing.T) {
|
||||
kv := newMemKV()
|
||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||
raw, rev, _, _ := kv.Get(ctx, id)
|
||||
cur, _ := DecodeTry(raw)
|
||||
ended := time.Now()
|
||||
cur.State, cur.Ended = TryPass, ended
|
||||
data, _ := encodeTry(cur)
|
||||
if _, err := kv.Update(ctx, id, data, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := ended.Add(TryKeep - time.Hour)
|
||||
tr := &Tryer{Tries: kv, Now: func() time.Time { return now }}
|
||||
tr.Reconsider(ctx)
|
||||
if _, _, found, _ := kv.Get(ctx, id); !found {
|
||||
t.Fatal("the try was removed before TryKeep")
|
||||
}
|
||||
now = ended.Add(TryKeep + time.Hour)
|
||||
tr.Reconsider(ctx)
|
||||
if _, _, found, _ := kv.Get(ctx, id); found {
|
||||
t.Error("the try was kept past TryKeep")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -130,14 +130,40 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
||||
if m.SecretsOwner != "mesh-delivery-checker" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3).
|
||||
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**"}}}; !reflect.DeepEqual(m.Streams, want) {
|
||||
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3). A whole try says try.started with its members and try.ended, so the indexer writes and deletes the candidate of its group (issue 1921).
|
||||
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}}; !reflect.DeepEqual(m.Streams, want) {
|
||||
t.Errorf("streams = %+v, want %+v", m.Streams, want)
|
||||
}
|
||||
// Only the checker's account publishes try.* on its stream checks: the stream is declared by its owner alone, and no other
|
||||
// module of the repository declares a stream carrying a try event (the indexer and the others only consume them).
|
||||
siblings, err := filepath.Glob("../*/module.json")
|
||||
if err != nil || len(siblings) < 2 {
|
||||
t.Fatalf("the sibling manifests = %v, %v", siblings, err)
|
||||
}
|
||||
for _, f := range siblings {
|
||||
raw, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var other struct {
|
||||
Module string `json:"module"`
|
||||
Streams []stream `json:"streams"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &other); err != nil {
|
||||
t.Fatalf("%s: %v", f, err)
|
||||
}
|
||||
for _, st := range other.Streams {
|
||||
for _, e := range st.Events {
|
||||
if strings.HasPrefix(e, "try.") && other.Module != m.Module {
|
||||
t.Errorf("%s declares the stream %s with %s: only the checker's account publishes try events", other.Module, st.Name, e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
|
||||
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check
|
||||
// is asked again by hand (checker_ask_again).
|
||||
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code.
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"mesh-build-node.toolchain.reported"
|
||||
],
|
||||
"streams": [
|
||||
{ "name": "checks", "events": ["preview.**", "owed.**"] }
|
||||
{ "name": "checks", "events": ["preview.**", "owed.**", "try.**"] }
|
||||
],
|
||||
"state": [
|
||||
{ "name": "asks", "history": 1 },
|
||||
@@ -56,6 +56,7 @@
|
||||
"checker_previews",
|
||||
"checker_pull",
|
||||
"checker_status",
|
||||
"checker_tries",
|
||||
"checker_try",
|
||||
"checker_try_end",
|
||||
"checker_try_read",
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
// Command mesh-delivery-forge-bridge-tools serves the forge bridge's tools (novox/hq ADR 0633 §4, issue 1585): the runtime
|
||||
// launches it for a call. bridge_groups reads the delivery groups the gate holds and what the bridge kept of their ends;
|
||||
// bridge_group_end_again writes an ask into the bridge's state end-asks and waits for the bridge's answer there, since the
|
||||
// bridge is the other program and serves no request subject (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
"mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// Tool names.
|
||||
const (
|
||||
GroupsTool = "bridge_groups"
|
||||
EndAgainTool = "bridge_group_end_again"
|
||||
// MergedTool answers the merged message of a merge on request (issue 1585).
|
||||
MergedTool = "bridge_merged_message"
|
||||
)
|
||||
|
||||
// asker writes an end-asks value and waits for the bridge's answer to it.
|
||||
type asker func(key string, value []byte) (groups.EndAsk, error)
|
||||
|
||||
// Tools are the bridge's tools: groupsRead reads the groups the gate holds with what the bridge kept of them.
|
||||
func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string) (json.RawMessage, error)) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) }
|
||||
return []stdio.Tool{
|
||||
{Name: MergedTool,
|
||||
Description: "The merged message of one merge, asked again (issue 1585): the message the bridge wrote on its stream FORGE when the " +
|
||||
"pull request was merged into the trunk, built the same way from the forge (the merge commit's trees and the manifests " +
|
||||
"and lock files of its module directories) and from the gate's merges state (the pull request's number, head and " +
|
||||
"group). For the indexer, which asks it when it takes in a held merge it does not hold in memory (a restart, a new " +
|
||||
"build), and the ask-again a lost message lacked. About 0.5 MB for a repository of many modules. Answers {message} " +
|
||||
"with the forge message whole, or why it cannot: the gate holds no merge with that commit, or the forge did not answer.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "merge"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "the repository, owner/name, as the forge names it"},
|
||||
"merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
raw, err := merged(str(args, "repository"), str(args, "merge"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"message": raw}, nil
|
||||
}},
|
||||
{Name: GroupsTool,
|
||||
Description: "The delivery groups the gate holds (more than one member), each with its members' merge state and whether each " +
|
||||
"merged message is written on the forge stream, and whether the group's end (forge.group.merged) is written: ended, " +
|
||||
"or open with why (gathered, or merged and its end not written yet). An open group with every member merged is " +
|
||||
"what bridge_group_end_again says the end of. Each group names its last end ask (who, why, when) and the bridge's " +
|
||||
"answer to it, from the state end-asks. A group's name narrows it to that group; open lists only open ones.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "only this group, by the name the gate keys it"},
|
||||
"open": map[string]any{"type": "boolean", "description": "only the groups whose end is not written"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
rows, unread, err := groupsRead()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
only, _ := args["open"].(bool)
|
||||
group := str(args, "group")
|
||||
kept := rows[:0]
|
||||
for _, r := range rows {
|
||||
if (group != "" && r.Group != group) || (only && !r.Open) {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, r)
|
||||
}
|
||||
return map[string]any{"groups": kept, "unread": unread}, nil
|
||||
}},
|
||||
{Name: EndAgainTool,
|
||||
Description: "Say a delivery group's end (forge.group.merged) again, by hand, for a group whose end the indexer did not take " +
|
||||
"in (a group bridge_groups names as merged with its end not written, or one the indexer waits for). The bridge " +
|
||||
"writes it again from the gate's merges, with the time appended to the message id so the stream does not drop it as " +
|
||||
"a duplicate; the indexer takes an end for a group it already took in without changing anything. Refused while " +
|
||||
"a member is not merged or its merged message is not written. Answers whether it was written.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{
|
||||
"group": map[string]any{"type": "string", "description": "the group, by the name the gate keys it"},
|
||||
"why": map[string]any{"type": "string", "description": "why its end is said again, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
key, value, err := groups.NewEndAsk(str(args, "group"), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"group": a.Group, "written": a.Written, "answer": a.Answer, "by": a.By, "why": a.Why}, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
// MergedOf is the merged message of the merge of repository at commit merge, as the bridge writes it: the pull request's
|
||||
// number, head and group from the gate's merges (raw, by key), the rest from the forge through trees and files
|
||||
// (bridge.MergedMessage), and, for a member of a delivery group, the group and its members as the process sets them.
|
||||
func MergedOf(raws map[string][]byte, repository, merge string, trees bridge.Trees, files bridge.Files) (json.RawMessage, error) {
|
||||
owner, repo, ok := strings.Cut(repository, "/")
|
||||
if !ok || owner == "" || repo == "" {
|
||||
return nil, fmt.Errorf("%q is not owner/name", repository)
|
||||
}
|
||||
for _, key := range slices.Sorted(maps.Keys(raws)) {
|
||||
m, err := delivery.DecodeMerge(raws[key])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool {
|
||||
return delivery.SameRepository(p.Repository, repository) && p.Commit == merge
|
||||
})
|
||||
if at < 0 {
|
||||
continue
|
||||
}
|
||||
p := m.Pulls[at]
|
||||
msg, err := bridge.MergedMessage(owner, repo, p.Number, m.Group, p.Head.Head, merge, trees, files)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the merged message of %s at %s cannot be built: %w", repository, merge, err)
|
||||
}
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, approved := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; approved && len(m.Approved) >= 2 {
|
||||
f.Group, f.Members = m.Group, groups.MembersOf(m)
|
||||
}
|
||||
out, err := forgemsg.EncodeForge(f)
|
||||
return out, err
|
||||
}
|
||||
return nil, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, so its pull request is not known", repository, merge)
|
||||
}
|
||||
|
||||
// read is the groups the gate holds with what the bridge kept of them: their ends in performed, and the last ask of
|
||||
// each in end-asks with the bridge's answer.
|
||||
func read(merges, performed, asks stdio.KeptState) ([]groups.Row, []string, error) {
|
||||
keys, err := merges.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the gate's merges cannot be listed: %w", err)
|
||||
}
|
||||
raws := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
e, err := merges.Get(k)
|
||||
if err != nil || e == nil {
|
||||
continue
|
||||
}
|
||||
raws[k] = e.Value
|
||||
}
|
||||
pkeys, err := performed.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err)
|
||||
}
|
||||
published, ended := map[string]bool{}, map[string]uint64{}
|
||||
for _, k := range pkeys {
|
||||
switch {
|
||||
case strings.HasPrefix(k, "published."):
|
||||
published[k] = true
|
||||
case strings.HasPrefix(k, "groups."):
|
||||
if e, err := performed.Get(k); err == nil && e != nil {
|
||||
rev, _ := strconv.ParseUint(strings.TrimSpace(string(e.Value)), 10, 64)
|
||||
ended[strings.TrimPrefix(k, "groups.")] = rev
|
||||
}
|
||||
}
|
||||
}
|
||||
akeys, err := asks.Keys()
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the bridge's state %s cannot be listed: %w", groups.EndAsksState, err)
|
||||
}
|
||||
askRaws := map[string][]byte{}
|
||||
for _, k := range akeys {
|
||||
if e, err := asks.Get(k); err == nil && e != nil {
|
||||
askRaws[k] = e.Value
|
||||
}
|
||||
}
|
||||
rows, unread := groups.Rows(raws, published, ended, askRaws)
|
||||
return rows, unread, nil
|
||||
}
|
||||
|
||||
// askEnd writes the end-asks value at key and waits, on a watch of the key, at most bound for the bridge's answer.
|
||||
func askEnd(s stdio.KeptState, bound time.Duration) asker {
|
||||
return func(key string, value []byte) (groups.EndAsk, error) {
|
||||
answered := make(chan groups.EndAsk, 1)
|
||||
var rev uint64
|
||||
var mu sync.Mutex
|
||||
take := func(revision uint64, v []byte) {
|
||||
mu.Lock()
|
||||
mine := rev
|
||||
mu.Unlock()
|
||||
if mine == 0 || revision <= mine {
|
||||
return
|
||||
}
|
||||
if a, err := groups.DecodeEndAsk(v); err == nil && a.Done() {
|
||||
select {
|
||||
case answered <- a:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := s.Watch(key, func(c stdio.StateChange) error {
|
||||
if c.Op == "put" {
|
||||
take(c.Revision, c.Value)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return groups.EndAsk{}, fmt.Errorf("watching %s.%s: %w", groups.EndAsksState, key, err)
|
||||
}
|
||||
r, err := s.Put(key, json.RawMessage(value))
|
||||
if err != nil {
|
||||
return groups.EndAsk{}, fmt.Errorf("the ask could not be written to %s: %w", groups.EndAsksBucket, err)
|
||||
}
|
||||
mu.Lock()
|
||||
rev = r
|
||||
mu.Unlock()
|
||||
if e, err := s.Get(key); err == nil && e != nil {
|
||||
take(e.Revision, e.Value)
|
||||
}
|
||||
wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8)
|
||||
defer cancel()
|
||||
select {
|
||||
case a := <-answered:
|
||||
return a, nil
|
||||
case <-wait.Done():
|
||||
return groups.EndAsk{}, fmt.Errorf("the bridge did not answer within %s: its answer will be in %s.%s", bound, groups.EndAsksBucket, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// askCaller calls a tool as this bundle's module, through the runtime (stdio.Ask), the module's own invokes granting it.
|
||||
func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) {
|
||||
return stdio.Ask(key, body)
|
||||
}
|
||||
|
||||
func main() {
|
||||
merges, performed, asks := stdio.State(delivery.GateModule+".merges"), stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState)
|
||||
tools := Tools(askEnd(asks, 25*time.Second), time.Now,
|
||||
func() ([]groups.Row, []string, error) { return read(merges, performed, asks) },
|
||||
func(repository, merge string) (json.RawMessage, error) {
|
||||
keys, err := merges.Keys()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the gate's merges cannot be listed: %w", err)
|
||||
}
|
||||
raws := map[string][]byte{}
|
||||
for _, k := range keys {
|
||||
if e, err := merges.Get(k); err == nil && e != nil {
|
||||
raws[k] = e.Value
|
||||
}
|
||||
}
|
||||
ctx := context.Background()
|
||||
return MergedOf(raws, repository, merge, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller))
|
||||
})
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// 1585/T18, T19 (ADR 0633 §4): bridge_groups names each group the gate holds with its members' state and whether its end is
|
||||
// written (open or ended, with why), narrowed by group and by open; bridge_group_end_again writes an ask naming the group,
|
||||
// who asked and why, and answers what the bridge answered.
|
||||
func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) {
|
||||
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
|
||||
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
|
||||
enc := func(group string, state2 delivery.MergeState, c string) []byte {
|
||||
v, err := delivery.EncodeMerge(delivery.Merge{Group: group, Ask: "ask", Approved: map[string]string{"novox/a@1": h1, "novox/b@2": h2},
|
||||
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 1, Head: h1}, State: delivery.MergeMerged, Commit: c1},
|
||||
{Head: delivery.Head{Repository: "novox/b", Number: 2, Head: h2}, State: state2, Commit: c}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
lone, _ := delivery.EncodeMerge(delivery.Merge{Group: "solo", Ask: "ask", Approved: map[string]string{"novox/a@5": h1},
|
||||
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 5, Head: h1}, State: delivery.MergeMerged, Commit: c1}}})
|
||||
merges := map[string][]byte{"ended": enc("ended", delivery.MergeMerged, c2), "ready": enc("ready", delivery.MergeMerged, c2),
|
||||
"gathered": enc("gathered", delivery.MergeMerging, ""), "solo": lone, "bad": []byte("{")}
|
||||
performed := map[string]bool{}
|
||||
for _, c := range []string{c1, c2} {
|
||||
performed[groups.PublishedKey("novox/a", 1, c)], performed[groups.PublishedKey("novox/b", 2, c)] = true, true
|
||||
}
|
||||
// The state end-asks keeps the last ask of the group ready, answered, and one that does not read.
|
||||
answeredAsk, _ := json.Marshal(groups.EndAsk{V: 1, Group: "ready", Why: "lost", By: "lead@shanks", At: time.Date(2026, 11, 4, 22, 0, 0, 0, time.UTC),
|
||||
Written: true, Answer: "the end of the group is written again", Answered: time.Date(2026, 11, 4, 22, 0, 1, 0, time.UTC)})
|
||||
asks := map[string][]byte{"ready": answeredAsk, "gathered": []byte("{")}
|
||||
var asked groups.EndAsk
|
||||
var askedKey string
|
||||
ask := func(key string, value []byte) (groups.EndAsk, error) {
|
||||
askedKey = key
|
||||
a, err := groups.DecodeEndAsk(value)
|
||||
a.Written, a.Answer, a.Answered = true, "the end of the group is written again", time.Now()
|
||||
asked = a
|
||||
return a, err
|
||||
}
|
||||
tools := Tools(ask, func() time.Time { return time.Date(2026, 11, 4, 23, 0, 0, 0, time.UTC) },
|
||||
func() ([]groups.Row, []string, error) {
|
||||
rows, unread := groups.Rows(merges, performed, map[string]uint64{"ended": 7}, asks)
|
||||
return rows, unread, nil
|
||||
}, nil)
|
||||
call := func(name string, args map[string]any) any {
|
||||
for _, tl := range tools {
|
||||
if tl.Name == name {
|
||||
out, err := tl.Run(args)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", name, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", name)
|
||||
return nil
|
||||
}
|
||||
rowsOf := func(args map[string]any) map[string]groups.Row {
|
||||
raw, _ := json.Marshal(call(GroupsTool, args))
|
||||
var out struct {
|
||||
Groups []groups.Row `json:"groups"`
|
||||
Unread []string `json:"unread"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := map[string]groups.Row{}
|
||||
for _, r := range out.Groups {
|
||||
m[r.Group] = r
|
||||
}
|
||||
if len(args) == 0 && strings.Join(out.Unread, " ") != "bad end-asks.gathered" {
|
||||
t.Errorf("unread = %v, want bad and end-asks.gathered", out.Unread)
|
||||
}
|
||||
return m
|
||||
}
|
||||
all := rowsOf(map[string]any{})
|
||||
if len(all) != 3 || !all["ended"].Ended || all["ended"].EndedAt != 7 {
|
||||
t.Fatalf("groups = %+v", all)
|
||||
}
|
||||
if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published {
|
||||
t.Errorf("ready = %+v, want open, merged and published, its end the thing to say", r)
|
||||
}
|
||||
if a := all["ready"].LastEndAsk; a == nil || a.By != "lead@shanks" || a.Why != "lost" || !a.Written || a.Answer == "" || all["ended"].LastEndAsk != nil {
|
||||
t.Errorf("ready's last end ask = %+v, ended's %+v: want ready's ask and its answer, none for ended", a, all["ended"].LastEndAsk)
|
||||
}
|
||||
if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != string(delivery.MergeMerging) {
|
||||
t.Errorf("gathered = %+v", r)
|
||||
}
|
||||
if open := rowsOf(map[string]any{"open": true}); len(open) != 2 || open["ended"].Group != "" {
|
||||
t.Errorf("open = %+v, want ready and gathered", open)
|
||||
}
|
||||
if one := rowsOf(map[string]any{"group": "ready"}); len(one) != 1 {
|
||||
t.Errorf("group ready = %+v", one)
|
||||
}
|
||||
out := call(EndAgainTool, map[string]any{"group": "ready", "why": "the indexer never took it in"}).(map[string]any)
|
||||
if askedKey != "ready" || asked.Group != "ready" || asked.Why != "the indexer never took it in" || out["written"] != true {
|
||||
t.Errorf("the ask %q %+v, answered %v", askedKey, asked, out)
|
||||
}
|
||||
for _, tl := range tools {
|
||||
if tl.Name == EndAgainTool {
|
||||
if _, err := tl.Run(map[string]any{}); err == nil {
|
||||
t.Error("an ask naming no group was taken")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1585: bridge_merged_message answers the merged message as the bridge publishes it: for a lone pull request the same bytes
|
||||
// the event gives through Convert, for a member of a delivery group those with its group and members set as the process
|
||||
// sets them; a merge the gate does not hold, or a forge that does not answer, is said.
|
||||
t.Run("bridge_merged_message answers the merged message as published", func(t *testing.T) {
|
||||
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
|
||||
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
|
||||
trees := func(owner, repo, sha string) (string, map[string]string, error) {
|
||||
return "root-" + sha[:4], map[string]string{"": "r", "modules/a": "ta-" + sha[:4]}, nil
|
||||
}
|
||||
files := func(owner, repo, path, ref string) ([]byte, error) {
|
||||
if strings.HasSuffix(path, "module.json") {
|
||||
return []byte(`{"module":"a"}`), nil
|
||||
}
|
||||
return nil, bridge.ErrNotFound
|
||||
}
|
||||
published := func(repo string, number int, head, branch, merge string) forgemsg.Forge {
|
||||
body, _ := json.Marshal(map[string]any{"owner": "novox", "repo": repo, "number": number, "head": branch, "head_sha": head, "base": "main", "merge_commit_sha": merge})
|
||||
msg, err := bridge.Convert(bridge.MergedSubject, body, trees, files)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
lone, _ := delivery.EncodeMerge(delivery.Merge{Group: "solo", Ask: "a", Approved: map[string]string{"novox/a@5": h1},
|
||||
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 5, Head: h1}, State: delivery.MergeMerged, Commit: c1}}})
|
||||
group, _ := delivery.EncodeMerge(delivery.Merge{Group: "feat/g", Ask: "a", Approved: map[string]string{"novox/a@6": h1, "novox/b@7": h2},
|
||||
Pulls: []delivery.MergePull{{Head: delivery.Head{Repository: "novox/a", Number: 6, Head: h1}, State: delivery.MergeMerged, Commit: c1},
|
||||
{Head: delivery.Head{Repository: "novox/b", Number: 7, Head: h2}, State: delivery.MergeMerging}}})
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
raws map[string][]byte
|
||||
repository string
|
||||
merge string
|
||||
want forgemsg.Forge
|
||||
fail string
|
||||
}{
|
||||
{name: "a lone pull request", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c1, want: published("a", 5, h1, "solo", c1)},
|
||||
{name: "a member of a group", raws: map[string][]byte{"feat~g": group}, repository: "novox/a", merge: c1, want: func() forgemsg.Forge {
|
||||
f := published("a", 6, h1, "feat/g", c1)
|
||||
f.Group, f.Members = "feat/g", []delivery.Head{{Repository: "novox/a", Number: 6, Head: h1}, {Repository: "novox/b", Number: 7, Head: h2}}
|
||||
return f
|
||||
}()},
|
||||
{name: "a merge the gate does not hold", raws: map[string][]byte{"solo": lone, "bad": []byte("{")}, repository: "novox/a", merge: c2, fail: "hold no merge"},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
raw, err := MergedOf(c.raws, c.repository, c.merge, trees, files)
|
||||
if c.fail != "" {
|
||||
if err == nil || !strings.Contains(err.Error(), c.fail) {
|
||||
t.Fatalf("err %v, want %q", err, c.fail)
|
||||
}
|
||||
return
|
||||
}
|
||||
want, _ := forgemsg.EncodeForge(c.want)
|
||||
if err != nil || !bytes.Equal(raw, want) {
|
||||
t.Fatalf("err %v, answer\n%s\nwant the published message\n%s", err, raw, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := MergedOf(map[string][]byte{"solo": lone}, "novox/a", c1, func(string, string, string) (string, map[string]string, error) {
|
||||
return "", nil, errors.New("the forge does not answer")
|
||||
}, files); err == nil || !strings.Contains(err.Error(), "does not answer") {
|
||||
t.Errorf("a forge that does not answer: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -66,58 +65,12 @@ func callerOn(nc *nats.Conn, user string) caller {
|
||||
}
|
||||
}
|
||||
|
||||
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner,
|
||||
// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4).
|
||||
const commitTreesTool = "gitea.gitea_commit_trees"
|
||||
|
||||
// treesVia asks the forge for a commit's trees through call.
|
||||
func treesVia(ctx context.Context, call caller) bridge.Trees {
|
||||
return func(owner, repo, sha string) (string, map[string]string, error) {
|
||||
raw, err := call(ctx, commitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var a struct {
|
||||
Root string `json:"root"`
|
||||
Trees map[string]string `json:"trees"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
|
||||
}
|
||||
if a.Root == "" {
|
||||
return "", nil, fmt.Errorf("the answer of %s names no root tree", commitTreesTool)
|
||||
}
|
||||
return a.Root, a.Trees, nil
|
||||
}
|
||||
return bridge.TreesVia(ctx, bridge.Caller(call))
|
||||
}
|
||||
|
||||
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
|
||||
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
|
||||
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
|
||||
const getFileTool = "gitea.gitea_get_file"
|
||||
|
||||
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
|
||||
// recognises it as /: 404 /).
|
||||
var notFound = regexp.MustCompile(`: 404 `)
|
||||
|
||||
// filesVia asks the forge for a file through call. A file the forge says is not there is bridge.ErrNotFound.
|
||||
func filesVia(ctx context.Context, call caller) bridge.Files {
|
||||
return func(owner, repo, path, ref string) ([]byte, error) {
|
||||
raw, err := call(ctx, getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
|
||||
if err != nil {
|
||||
if notFound.MatchString(err.Error()) {
|
||||
return nil, fmt.Errorf("%w: %v", bridge.ErrNotFound, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
|
||||
}
|
||||
return []byte(*a.File.Content), nil
|
||||
}
|
||||
return bridge.FilesVia(ctx, bridge.Caller(call))
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -120,8 +121,8 @@ func TestARefusedActIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1341/AC
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.refused" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
m, err := delivery.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != delivery.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" {
|
||||
m, err := forgemsg.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != forgemsg.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" {
|
||||
t.Errorf("message %+v err %v", m, err)
|
||||
}
|
||||
f.refuse = nil
|
||||
@@ -188,8 +189,8 @@ func TestARefusedMergeIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1342/
|
||||
if len(f.calls) != 1 || len(pub.data) != 1 {
|
||||
t.Fatalf("calls %d messages %d, want one each", len(f.calls), len(pub.data))
|
||||
}
|
||||
m, err := delivery.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != delivery.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA ||
|
||||
m, err := forgemsg.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != forgemsg.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA ||
|
||||
m.Act != "merge" || m.Words != "novox/mesh-delivery#12 is not merged: its head moved" {
|
||||
t.Errorf("message %+v err %v", m, err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// endAsked answers one value of the state end-asks (the tool bridge_group_end_again, ADR 0633 §4): a value not yet answered
|
||||
// says the group's end again from the gate's merges, with the ask's time appended to the message id so the stream's duplicate
|
||||
// window does not drop it, and is written back by compare-and-set at its revision with whether it was written. A value
|
||||
// answered already (this bridge's own write) is left.
|
||||
func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte) {
|
||||
a, err := groups.DecodeEndAsk(value)
|
||||
if err != nil {
|
||||
g.logger.Printf("%s.%s holds what is not an end ask, left: %v", groups.EndAsksState, key, err)
|
||||
return
|
||||
}
|
||||
if a.Done() {
|
||||
return
|
||||
}
|
||||
a = g.answerEnd(ctx, key, a, time.Now())
|
||||
g.logger.Printf("the end of the group %q said again by %s (%s): %s", a.Group, a.By, a.Why, a.Answer)
|
||||
data, err := json.Marshal(a)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
g.logger.Printf("%s.%s: the answer could not be written: %v", groups.EndAsksState, key, err)
|
||||
}
|
||||
}
|
||||
|
||||
// readMerges reads one key of the gate's merges state.
|
||||
func (g *grouper) readMerges(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
if g.merges == nil {
|
||||
return nil, 0, false, errors.New("the bridge does not read the gate's merges (its bucket was not there at the start)")
|
||||
}
|
||||
value, rev, found, err := g.merges(ctx, key)
|
||||
return value, rev, found, err
|
||||
}
|
||||
|
||||
// answerEnd says the group's end again for an ask and returns the ask with its answer.
|
||||
func (g *grouper) answerEnd(ctx context.Context, key string, a groups.EndAsk, at time.Time) groups.EndAsk {
|
||||
a.Answered = at.UTC()
|
||||
switch merges, mrev, found, err := g.readMerges(ctx, key); {
|
||||
case err != nil:
|
||||
a.Answer = "the gate's merges " + key + " cannot be read: " + err.Error()
|
||||
case !found:
|
||||
a.Answer = "the gate holds no merge of the group " + a.Group
|
||||
case g.end(ctx, key, mrev, merges, groups.AgainStamp(a.At)):
|
||||
a.Written, a.Answer = true, "the end of the group is written again"
|
||||
default:
|
||||
a.Answer = "the end is not written: a member is not merged or its merged message is not written, or FORGE refused it (the bridge's journal says which)"
|
||||
}
|
||||
return a
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"slices"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
groupkeys "mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// A delivery group's merges reach the indexer together (novox/hq ADR 0633 §4): the bridge sets the group on each
|
||||
// member's merged message, from the gate's merges state it reads (ADR 0437 §8), and once every approved member is
|
||||
// merged and its merged message written on FORGE it says the group's end, forge.group.merged, once.
|
||||
//
|
||||
// What it keeps of it is in its state performed, beside the gate's keys it acted on: `published.<hash>` for each
|
||||
// member's merged message written (the hash is the first 16 hex digits of the sha256 of `<repository>#<number>@<merge>`),
|
||||
// and `groups.<merges key>` once the group's end is written, its value the revision of the gate's merges key it was
|
||||
// worked out from. A group whose end is not written yet is open (bridge_groups reads them).
|
||||
|
||||
// merges reads one key of the gate's merges state.
|
||||
type mergesRead func(ctx context.Context, key string) (value []byte, rev uint64, found bool, err error)
|
||||
|
||||
// grouper sets a member's group and says a group's end.
|
||||
type grouper struct {
|
||||
merges mergesRead
|
||||
send func(ctx context.Context, subject string, data []byte, id string) error // writes on FORGE with the message id
|
||||
done revisions
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// sendVia writes on FORGE through pub, with the message id.
|
||||
func sendVia(pub publisher) func(context.Context, string, []byte, string) error {
|
||||
return func(ctx context.Context, subject string, data []byte, id string) error {
|
||||
_, err := pub.Publish(ctx, subject, data, jetstream.WithMsgID(id))
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
func kvMerges(kv jetstream.KeyValue) mergesRead {
|
||||
return func(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// groupOf is the gate's merge of the group a merged message's branch names, when the message's pull request is one of
|
||||
// its approved members and the group has more than one: a lone pull request is no group.
|
||||
func (g *grouper) groupOf(ctx context.Context, f forgemsg.Forge) (delivery.Merge, uint64, bool) {
|
||||
if g == nil || g.merges == nil || f.Branch == "" {
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
key, err := delivery.MergesKey(f.Branch)
|
||||
if err != nil {
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
value, rev, found, err := g.merges(ctx, key)
|
||||
if err != nil {
|
||||
g.logger.Printf("%s#%d: the gate's merges.%s cannot be read, so its merged message names no group: %v", f.Repository, f.Number, key, err)
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
if !found {
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
m, err := delivery.DecodeMerge(value)
|
||||
if err != nil {
|
||||
g.logger.Printf("%s#%d: the gate's merges.%s is not a merge, so its merged message names no group: %v", f.Repository, f.Number, key, err)
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
if _, ok := m.Approved[delivery.ApprovedKey(f.Repository, f.Number)]; !ok || len(m.Approved) < 2 {
|
||||
return delivery.Merge{}, 0, false
|
||||
}
|
||||
return m, rev, true
|
||||
}
|
||||
|
||||
// member sets the group and its approved members on a merged message's body (ADR 0633 §4); any other body, and a
|
||||
// merge of no group, is left as it is.
|
||||
func (g *grouper) member(ctx context.Context, data []byte) ([]byte, forgemsg.Forge, bool) {
|
||||
f, err := forgemsg.DecodeForge(data)
|
||||
if err != nil || f.Kind != forgemsg.ForgeMerged {
|
||||
return data, f, false
|
||||
}
|
||||
m, _, ok := g.groupOf(ctx, f)
|
||||
if !ok {
|
||||
return data, f, false
|
||||
}
|
||||
f.Group, f.Members = m.Group, groupkeys.MembersOf(m)
|
||||
out, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.Group, err)
|
||||
return data, f, false
|
||||
}
|
||||
return out, f, true
|
||||
}
|
||||
|
||||
// published keeps that a member's merged message is written, and says the group's end when it was the last.
|
||||
func (g *grouper) published(ctx context.Context, f forgemsg.Forge) {
|
||||
if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number, f.Merge), 1); err != nil {
|
||||
g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err)
|
||||
return
|
||||
}
|
||||
key, err := delivery.MergesKey(f.Branch)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
value, rev, found, err := g.merges(ctx, key)
|
||||
if err != nil || !found {
|
||||
return
|
||||
}
|
||||
g.ended(ctx, key, rev, value)
|
||||
}
|
||||
|
||||
// ended says the group's end (forge.group.merged) once every approved member of the gate's merge is merged and its
|
||||
// merged message written, after the last of them; never on a refusal (ADR 0441 §3: a held group merges on from it).
|
||||
// It is written once per group: `groups.<key>` in performed says it was. again says it is written again by hand
|
||||
// (bridge_group_end_again), with the time appended to the message id, so the duplicate window does not drop it.
|
||||
func (g *grouper) ended(ctx context.Context, key string, rev uint64, value []byte) {
|
||||
g.end(ctx, key, rev, value, "")
|
||||
}
|
||||
|
||||
func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, again string) bool {
|
||||
m, err := delivery.DecodeMerge(value)
|
||||
if err != nil || len(m.Approved) < 2 {
|
||||
return false
|
||||
}
|
||||
if again == "" {
|
||||
if _, seen, err := g.done.Get(ctx, groupkeys.Key(key)); err == nil && seen {
|
||||
return false
|
||||
}
|
||||
}
|
||||
end := forgemsg.ForgeGroupEnd{Group: m.Group}
|
||||
for _, member := range groupkeys.MembersOf(m) {
|
||||
at := slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool {
|
||||
return delivery.SameRepository(p.Repository, member.Repository) && p.Number == member.Number
|
||||
})
|
||||
if at < 0 || m.Pulls[at].Commit == "" {
|
||||
return false // not every member is merged (a merge commit is written only for a merged pull): the group stays gathered
|
||||
}
|
||||
if _, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok {
|
||||
return false // its merged message is not written yet: said when it is (published)
|
||||
}
|
||||
end.Members = append(end.Members, forgemsg.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit})
|
||||
}
|
||||
// The members in the order the gate merged them, so the last commit names the end.
|
||||
slices.SortStableFunc(end.Members, func(a, b forgemsg.GroupMerge) int {
|
||||
return slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == a.Merge }) -
|
||||
slices.IndexFunc(m.Pulls, func(p delivery.MergePull) bool { return p.Commit == b.Merge })
|
||||
})
|
||||
data, err := forgemsg.EncodeForgeGroupEnd(end)
|
||||
if err != nil {
|
||||
g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err)
|
||||
return false
|
||||
}
|
||||
id := groupkeys.EndID(m.Group, end.Members[len(end.Members)-1].Merge, "")
|
||||
if again != "" {
|
||||
id += "." + again
|
||||
}
|
||||
if err := g.send(ctx, forgeSubject(forgemsg.ForgeGroupEndEvent), data, id); err != nil {
|
||||
g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err)
|
||||
return false
|
||||
}
|
||||
if err := g.done.Put(ctx, groupkeys.Key(key), rev); err != nil {
|
||||
g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.Group, err)
|
||||
}
|
||||
g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.Group, len(end.Members), end.Members[len(end.Members)-1].Merge)
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
groupkeys "mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// idPub records the subjects, bodies and message ids it is handed.
|
||||
type idPub struct {
|
||||
subjects, ids []string
|
||||
data [][]byte
|
||||
}
|
||||
|
||||
func (p *idPub) send(_ context.Context, subject string, data []byte, id string) error {
|
||||
p.subjects, p.data, p.ids = append(p.subjects, subject), append(p.data, data), append(p.ids, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17, T19 (ADR 0633 §4)
|
||||
ctx := context.Background()
|
||||
h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40)
|
||||
c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40)
|
||||
merge := func(states ...delivery.MergePull) []byte {
|
||||
v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Pulls: states,
|
||||
Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
sdk := func(state delivery.MergeState, commit, words string) delivery.MergePull {
|
||||
return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-sdk", Number: 37, Head: h1}, State: state, Commit: commit, Words: words}
|
||||
}
|
||||
dlv := func(state delivery.MergeState, commit string) delivery.MergePull {
|
||||
return delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: state, Commit: commit}
|
||||
}
|
||||
gate := map[string][]byte{}
|
||||
var logs bytes.Buffer
|
||||
pub, done := &idPub{}, memRevisions{}
|
||||
g := &grouper{send: pub.send, done: done, logger: log.New(&logs, "", 0),
|
||||
merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := gate[key]
|
||||
return v, 7, ok, nil
|
||||
}}
|
||||
mergedOf := func(repo string, number int, head, commit string) []byte {
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
// A lone pull request's merged message carries no group.
|
||||
if _, f, ok := g.member(ctx, mergedOf("novox/hq", 4, h1, c1)); ok || f.Group != "" {
|
||||
t.Errorf("a lone pull request names the group %q", f.Group)
|
||||
}
|
||||
// The first member: its message names the group and both members; the group is not ended.
|
||||
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerging, ""))
|
||||
data, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1))
|
||||
if back, err := forgemsg.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 {
|
||||
t.Fatalf("a member's message: %+v %v", back, err)
|
||||
}
|
||||
g.published(ctx, f)
|
||||
// A refusal of the other member is no end (ADR 0441 §3: the group merges on from it later).
|
||||
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 9, Head: h2}, State: delivery.MergeRefused, Words: "405"})
|
||||
g.ended(ctx, "feat/take-in", 8, gate["feat/take-in"])
|
||||
// Merged on at the gate before its merged message is written: still no end; then its message ends the group.
|
||||
gate["feat/take-in"] = merge(sdk(delivery.MergeMerged, c1, ""), dlv(delivery.MergeMerged, c2))
|
||||
g.ended(ctx, "feat/take-in", 9, gate["feat/take-in"])
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Fatalf("an end before every member's merged message is written: %v", pub.subjects)
|
||||
}
|
||||
_, f, _ = g.member(ctx, mergedOf("novox/mesh-delivery", 9, h2, c2))
|
||||
g.published(ctx, f)
|
||||
g.ended(ctx, "feat/take-in", 10, gate["feat/take-in"]) // a later write of the gate says no second end
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.group.merged" || pub.ids[0] != "group-end.feat/take-in."+c2 {
|
||||
t.Fatalf("ends %v ids %v, want one forge.group.merged with id group-end.feat/take-in.%s", pub.subjects, pub.ids, c2)
|
||||
}
|
||||
end, err := forgemsg.DecodeForgeGroupEnd(pub.data[0])
|
||||
if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 {
|
||||
t.Errorf("the end = %+v %v, want both members in their merge order", end, err)
|
||||
}
|
||||
if rev, ok := done[groupkeys.Key("feat/take-in")]; !ok || rev != 7 {
|
||||
t.Errorf("performed %v: want groups.feat/take-in kept", done)
|
||||
}
|
||||
// Again by hand: written once more, its id not the first's.
|
||||
if !g.end(ctx, "feat/take-in", 10, gate["feat/take-in"], "20261104T2300") || pub.ids[1] == pub.ids[0] {
|
||||
t.Errorf("the end again by hand: ids %v", pub.ids)
|
||||
}
|
||||
// 1585/T19: an ask to say a group's end again is answered by the bridge from the gate's merges, with the ask's time on
|
||||
// the message id; not written while a member is not merged, with the reason; not for a group the gate does not hold.
|
||||
pull := func(repo string, n int, head string, state delivery.MergeState, commit string) delivery.MergePull {
|
||||
return delivery.MergePull{Head: delivery.Head{Repository: repo, Number: n, Head: head}, State: state, Commit: commit}
|
||||
}
|
||||
askMerge := func(second delivery.MergeState, commit2 string) []byte {
|
||||
v, err := delivery.EncodeMerge(delivery.Merge{Group: "feat/take-in", Ask: "ask", Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2},
|
||||
Pulls: []delivery.MergePull{pull("novox/mesh-sdk", 37, h1, delivery.MergeMerged, c1), pull("novox/mesh-delivery", 9, h2, second, commit2)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
ask := groupkeys.EndAsk{V: 1, Group: "feat/take-in", By: "operator@shanks", Why: "the indexer never took it in", At: time.Date(2026, 11, 4, 23, 0, 0, 123, time.UTC)}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
gate map[string][]byte
|
||||
published bool
|
||||
written bool
|
||||
answer string
|
||||
}{
|
||||
{"merged and published: written again", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, true, true, "written again"},
|
||||
{"a member's message not written", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerged, c2)}, false, false, "not written"},
|
||||
{"a member not merged", map[string][]byte{"feat/take-in": askMerge(delivery.MergeMerging, "")}, true, false, "not written"},
|
||||
{"no merge at the gate", map[string][]byte{}, true, false, "holds no merge"},
|
||||
} {
|
||||
pub, done := &idPub{}, memRevisions{}
|
||||
if tc.published {
|
||||
done[groupkeys.PublishedKey("novox/mesh-sdk", 37, c1)], done[groupkeys.PublishedKey("novox/mesh-delivery", 9, c2)] = 1, 1
|
||||
}
|
||||
g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0),
|
||||
merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := tc.gate[key]
|
||||
return v, 4, ok, nil
|
||||
}}
|
||||
got := g.answerEnd(ctx, "feat/take-in", ask, ask.At.Add(time.Second))
|
||||
if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] {
|
||||
t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids)
|
||||
}
|
||||
if tc.written && pub.ids[0] != groupkeys.EndID("feat/take-in", c2, groupkeys.AgainStamp(ask.At)) {
|
||||
t.Errorf("%s: the id %s does not carry the ask's time", tc.name, pub.ids[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
-13
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -36,7 +37,7 @@ import (
|
||||
//
|
||||
// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal
|
||||
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls,
|
||||
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
|
||||
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, performed, inputs and end-asks (ADR 0633 §5's hand end), and the stream forge; `**` is written `>` on the wire (modstreams.go
|
||||
// WirePattern). The consumer's durable is <node>_<module> on EVENTS (nats.go consumerDurable). The controller's own
|
||||
// user keeps all rights, as it has.
|
||||
const busConf = `
|
||||
@@ -83,6 +84,12 @@ accounts {
|
||||
"$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>"
|
||||
"$KV.mesh-delivery-forge-bridge_performed.>"
|
||||
"$KV.mesh-delivery-forge-bridge_inputs.>"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_end-asks"
|
||||
"$JS.FC.KV_mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$KV.mesh-delivery-forge-bridge_end-asks.>"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>"
|
||||
@@ -321,13 +328,13 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(raw.Data)
|
||||
f, err := forgemsg.DecodeForge(raw.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
|
||||
@@ -416,16 +423,16 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(raw.Data)
|
||||
f, err := forgemsg.DecodeForge(raw.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
|
||||
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
|
||||
// No mesh package: only the indexer computes what a fingerprint takes (ADR 0593 §2).
|
||||
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
select {
|
||||
@@ -478,7 +485,7 @@ func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err := delivery.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
|
||||
if f, err := forgemsg.DecodeForge(raw.Data); err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
|
||||
t.Errorf("%s: %+v err %v", repo, f, err)
|
||||
}
|
||||
}
|
||||
@@ -548,20 +555,20 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes
|
||||
t.Fatalf("FORGE holds %d messages (%v), want %d", info.State.Msgs, err, want)
|
||||
}
|
||||
}
|
||||
named := func(seq uint64) []delivery.OpenPull {
|
||||
named := func(seq uint64) []forgemsg.OpenPull {
|
||||
t.Helper()
|
||||
raw, err := forge.GetMsg(b.ctx, seq)
|
||||
if err != nil || raw.Subject != openSubject {
|
||||
t.Fatalf("message %d: %v %v, want the open message", seq, raw, err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(raw.Data)
|
||||
f, err := forgemsg.DecodeForge(raw.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f.Open
|
||||
}
|
||||
bothOpen := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
same := func(got []delivery.OpenPull) bool {
|
||||
bothOpen := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
same := func(got []forgemsg.OpenPull) bool {
|
||||
return len(got) == 2 && got[0] == bothOpen[0] && got[1] == bothOpen[1]
|
||||
}
|
||||
|
||||
@@ -713,7 +720,7 @@ func forgeMerged(t *testing.T, b *bed) []string {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if f, err := delivery.DecodeForge(raw.Data); err == nil && f.Kind == delivery.ForgeMerged {
|
||||
if f, err := forgemsg.DecodeForge(raw.Data); err == nil && f.Kind == forgemsg.ForgeMerged {
|
||||
out = append(out, f.Repository)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,9 +21,11 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
groupkeys "mesh-delivery-forge-bridge/internal/groups"
|
||||
)
|
||||
|
||||
// eventsStream is the shared stream every module's events are on; the module's consumer on it is the durable
|
||||
@@ -68,6 +70,12 @@ type publisher interface {
|
||||
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, logger *log.Logger) {
|
||||
handleGrouped(ctx, m, pub, trees, files, kept, nil, logger)
|
||||
}
|
||||
|
||||
// handleGrouped is handle; a merged message of a delivery group's member names its group (g, ADR 0633 §4; nil names
|
||||
// none), and the member's merged message written may end the group.
|
||||
func handleGrouped(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, kept bridge.Kept, g *grouper, logger *log.Logger) {
|
||||
msg, err := bridge.ConvertKept(m.Subject(), m.Data(), trees, files, kept)
|
||||
switch {
|
||||
case errors.Is(err, bridge.ErrRetry):
|
||||
@@ -88,10 +96,18 @@ func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees,
|
||||
if id := m.Headers().Get("x-event-id"); id != "" {
|
||||
opts = append(opts, jetstream.WithMsgID(id))
|
||||
}
|
||||
var member forgemsg.Forge
|
||||
grouped := false
|
||||
if g != nil {
|
||||
msg.Data, member, grouped = g.member(ctx, msg.Data)
|
||||
}
|
||||
if _, err := pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, opts...); err != nil {
|
||||
logger.Printf("%s: %s could not be written on FORGE, left for redelivery: %v", m.Subject(), msg.Event, err)
|
||||
return
|
||||
}
|
||||
if grouped {
|
||||
g.published(ctx, member)
|
||||
}
|
||||
if msg.Said != "" {
|
||||
logger.Printf("%s: %s", m.Subject(), msg.Said)
|
||||
}
|
||||
@@ -136,17 +152,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
|
||||
// A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the gate's merges.
|
||||
groups := &grouper{send: sendVia(js), done: kvRevisions{performedKV}, logger: logger}
|
||||
watching := make(chan error, 3)
|
||||
for _, w := range []struct {
|
||||
bucket string
|
||||
react func(ctx context.Context, key string, rev uint64, value []byte)
|
||||
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
|
||||
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, func(ctx context.Context, key string, rev uint64, value []byte) {
|
||||
perf.merges(ctx, key, rev, value)
|
||||
groups.ended(ctx, key, rev, value)
|
||||
}}} {
|
||||
// The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started
|
||||
// again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945).
|
||||
kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf)
|
||||
if kv == nil {
|
||||
continue
|
||||
}
|
||||
if w.bucket == delivery.MergesBucket {
|
||||
groups.merges = kvMerges(kv)
|
||||
}
|
||||
go func() {
|
||||
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
@@ -155,6 +179,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
})
|
||||
}()
|
||||
}
|
||||
// A person says a group's end again with bridge_group_end_again, which writes the state end-asks (ADR 0633 §4): watched, and
|
||||
// answered there; a state not there at this start is answered from the next start.
|
||||
if kv := process.OptionalBucket(ctx, js, groupkeys.EndAsksBucket, "bridge_group_end_again is not answered until the bridge starts again", logger.Printf); kv != nil {
|
||||
go func() {
|
||||
err := watch(ctx, kv, func(key string, rev uint64, value []byte) { groups.endAsked(ctx, kv, key, rev, value) })
|
||||
if err != nil && ctx.Err() == nil {
|
||||
logger.Printf("the end asks are no longer watched, so bridge_group_end_again is not answered until the bridge starts again: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err := said(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -171,7 +205,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Bounded: the turn is held, and no sync or other event goes on without it.
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, logger)
|
||||
handleGrouped(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), inputs, groups, logger)
|
||||
})
|
||||
}, logger.Printf)
|
||||
if err != nil {
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
@@ -85,7 +85,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
f, err := delivery.DecodeForge(pub.data[0])
|
||||
f, err := forgemsg.DecodeForge(pub.data[0])
|
||||
if err != nil || f.Number != 12 || f.Branch != "feat/x" || f.Head != head || f.Action != "updated" || !f.DirsKnown {
|
||||
t.Errorf("message %+v err %v", f, err)
|
||||
}
|
||||
@@ -107,11 +107,10 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
if !strings.Contains(logs.String(), "mesh.mod.gitea.event.pull.updated") || !strings.Contains(logs.String(), "head") {
|
||||
t.Errorf("log %q names neither the subject nor why", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
|
||||
// 1235: an unwritten message is not acked (a failed write is redelivered by the server, not retried here); merged here
|
||||
// from a test of its own, for the test budget.
|
||||
logs.Reset()
|
||||
m, pub = &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
|
||||
handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0))
|
||||
if m.acked != 0 || m.termd != 0 {
|
||||
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
|
||||
@@ -173,7 +172,7 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
f, err := delivery.DecodeForge(pub.data[0])
|
||||
f, err := forgemsg.DecodeForge(pub.data[0])
|
||||
if err != nil || f.Root != "r0" || f.Trees[""] != "t0" || f.Trees["modules/a"] != "t1" || f.Merge == "" {
|
||||
t.Errorf("message %+v err %v", f, err)
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
@@ -41,7 +41,7 @@ func (r *recorder) Publish(ctx context.Context, subject string, payload []byte,
|
||||
if err != nil {
|
||||
return ack, err
|
||||
}
|
||||
f, derr := delivery.DecodeForge(payload)
|
||||
f, derr := forgemsg.DecodeForge(payload)
|
||||
if derr != nil {
|
||||
return ack, nil
|
||||
}
|
||||
@@ -101,7 +101,7 @@ func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore,
|
||||
}
|
||||
var differing []bridge.Message
|
||||
for _, m := range msgs {
|
||||
f, err := delivery.DecodeForge(m.Data)
|
||||
f, err := forgemsg.DecodeForge(m.Data)
|
||||
if err != nil {
|
||||
differing = append(differing, m)
|
||||
continue
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds its entries inside {pulls}.
|
||||
@@ -43,7 +43,7 @@ func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1,
|
||||
}
|
||||
want := []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.pull"}
|
||||
for i, w := range want {
|
||||
f, err := delivery.DecodeForge(pub.data[i])
|
||||
f, err := forgemsg.DecodeForge(pub.data[i])
|
||||
if pub.subjects[i] != w || err != nil || f.Action != "updated" || f.DirsKnown || f.Branch != "feat/x" {
|
||||
t.Errorf("message %d on %s = %+v err %v", i, pub.subjects[i], f, err)
|
||||
}
|
||||
@@ -112,8 +112,8 @@ func TestSyncOpenWritesPullsForNewHeadsThenOneOpenMessageNamingAll(t *testing.T)
|
||||
if pub.subjects[1] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.open" {
|
||||
t.Fatalf("last message on %s, want the open message after the pull messages", pub.subjects[1])
|
||||
}
|
||||
f, err := delivery.DecodeForge(pub.data[1])
|
||||
want := []delivery.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
f, err := forgemsg.DecodeForge(pub.data[1])
|
||||
want := []forgemsg.OpenPull{{Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
if err != nil || len(f.Open) != 2 || f.Open[0] != want[0] || f.Open[1] != want[1] {
|
||||
t.Errorf("open message = %+v err %v, want it to name %v, the pull request whose head did not differ among them", f, err, want)
|
||||
}
|
||||
@@ -140,10 +140,10 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6
|
||||
pub := &fakePub{}
|
||||
heads := newHeads()
|
||||
r := &recorder{pub: pub, heads: heads, logger: log.New(&bytes.Buffer{}, "", 0)}
|
||||
write := func(f delivery.Forge) {
|
||||
write := func(f forgemsg.Forge) {
|
||||
t.Helper()
|
||||
ev, _ := delivery.ForgeEvent(f.Repository, f.Kind)
|
||||
raw, err := delivery.EncodeForge(f)
|
||||
ev, _ := forgemsg.ForgeEvent(f.Repository, f.Kind)
|
||||
raw, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -151,19 +151,19 @@ func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p := delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated}
|
||||
p := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated}
|
||||
write(p)
|
||||
if heads.get("novox.mesh-delivery.12") != head {
|
||||
t.Fatalf("bucket %v", heads.vals)
|
||||
}
|
||||
p.Action = delivery.PullClosed
|
||||
p.Action = forgemsg.PullClosed
|
||||
write(p)
|
||||
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
|
||||
t.Errorf("a closed pull request is still kept: %v", heads.vals)
|
||||
}
|
||||
p.Action = delivery.PullUpdated
|
||||
p.Action = forgemsg.PullUpdated
|
||||
write(p)
|
||||
write(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
|
||||
write(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
|
||||
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
|
||||
t.Errorf("a merged pull request is still kept: %v", heads.vals)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,413 @@
|
||||
// Package forgemsg is the forge bridge's contract on its own stream `forge` (novox/hq ADR 0633 §4): its messages (Forge,
|
||||
// one per pull request, merge, refusal or list of open pull requests) and a delivery group's end (ForgeGroupEnd). The
|
||||
// forge bridge writes them; the indexer and the gate read them. It is the forge bridge's own package: one module owns
|
||||
// these types, so they live with it and not in mesh-sdk (the operator, 2026-11-04).
|
||||
package forgemsg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Version is the version of every value this package writes and the only one it reads: mesh-sdk's delivery values'.
|
||||
const Version = delivery.Version
|
||||
|
||||
// ForgeStream is the forge bridge's stream as its manifest declares it; the broker derives the real names.
|
||||
const ForgeStream = "forge"
|
||||
|
||||
// The kinds of forge message, the last part of a forge event's name.
|
||||
const (
|
||||
ForgePull = "pull"
|
||||
ForgeMerged = "merged"
|
||||
ForgeRefused = "refused"
|
||||
ForgeOpen = "open"
|
||||
)
|
||||
|
||||
// OpenPull names an open pull request into the trunk: the repository as `<owner>/<name>` and the number.
|
||||
type OpenPull struct {
|
||||
Repository string `json:"repository"`
|
||||
Number int `json:"number"`
|
||||
}
|
||||
|
||||
// OpenEvent is the name of an open message: `forge.open`. It names no repository: it says every open pull request of
|
||||
// every repository the forge bridge reads (novox/hq ADR 0443 §5).
|
||||
func OpenEvent() string { return "forge." + ForgeOpen }
|
||||
|
||||
// The actions of a pull message.
|
||||
const (
|
||||
PullOpened = "opened"
|
||||
PullUpdated = "updated"
|
||||
PullClosed = "closed"
|
||||
)
|
||||
|
||||
// checkRepository refuses a repository that is not `<owner>/<name>`: exactly one slash, each part a valid name.
|
||||
func checkRepository(repository string) error {
|
||||
owner, name, ok := strings.Cut(repository, "/")
|
||||
if !ok || strings.Contains(name, "/") {
|
||||
return fmt.Errorf("delivery: the repository %q is not <owner>/<name>, with exactly one slash", repository)
|
||||
}
|
||||
if err := checkName("repository owner", owner); err != nil {
|
||||
return fmt.Errorf("delivery: the repository %q: %w", repository, err)
|
||||
}
|
||||
if err := checkName("repository name", name); err != nil {
|
||||
return fmt.Errorf("delivery: the repository %q: %w", repository, err)
|
||||
}
|
||||
if strings.Contains(owner, "/") {
|
||||
return fmt.Errorf("delivery: the repository %q is not <owner>/<name>, with exactly one slash", repository)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkForgeKind(kind string) error {
|
||||
switch kind {
|
||||
case ForgePull, ForgeMerged, ForgeRefused, ForgeOpen:
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("delivery: the forge kind %q is not pull, merged, refused or open", kind)
|
||||
}
|
||||
|
||||
// ForgeEvent is the name of a forge message of the given kind: `forge.<owner>.<name>.<kind>`.
|
||||
func ForgeEvent(repository, kind string) (string, error) {
|
||||
if err := checkForgeKind(kind); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if kind == ForgeOpen {
|
||||
return "", fmt.Errorf("delivery: an open message names no repository, its event is %s", OpenEvent())
|
||||
}
|
||||
if err := checkRepository(repository); err != nil {
|
||||
return "", err
|
||||
}
|
||||
owner, name, _ := strings.Cut(repository, "/")
|
||||
return "forge." + owner + "." + name + "." + kind, nil
|
||||
}
|
||||
|
||||
// ParseForgeEvent is the inverse of ForgeEvent.
|
||||
func ParseForgeEvent(event string) (repository, kind string, err error) {
|
||||
if event == OpenEvent() {
|
||||
return "", ForgeOpen, nil
|
||||
}
|
||||
parts := strings.Split(event, ".")
|
||||
if len(parts) != 4 || parts[0] != "forge" {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a forge event, forge.<owner>.<name>.<kind>", event)
|
||||
}
|
||||
repository = parts[1] + "/" + parts[2]
|
||||
if _, err := ForgeEvent(repository, parts[3]); err != nil {
|
||||
return "", "", fmt.Errorf("delivery: %q is not a forge event: %w", event, err)
|
||||
}
|
||||
return repository, parts[3], nil
|
||||
}
|
||||
|
||||
// Forge is a message of the forge bridge's stream, trimmed to repository, branch, pull request and head. A pull
|
||||
// message says Action; a merged message carries the merge commit, the tree of every directory holding a
|
||||
// module.json and the repository's root tree; a refused message names the owed act it answers and the forge's own
|
||||
// words. A pull message may also carry the module directories its head changes (Dirs, in the order the forge said) and
|
||||
// DirsKnown, true when the forge could say them: known and empty is DirsKnown with no Dirs, not known is neither
|
||||
// (the forge's module_dirs and module_dirs_said, mesh-catalog modules/gitea/trunk.ts). Other messages carry neither.
|
||||
type Forge struct {
|
||||
V int `json:"v"`
|
||||
Kind string `json:"kind"`
|
||||
Repository string `json:"repository"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Number int `json:"number,omitempty"`
|
||||
Head string `json:"head,omitempty"`
|
||||
Action string `json:"action,omitempty"`
|
||||
Dirs []string `json:"dirs,omitempty"`
|
||||
DirsKnown bool `json:"dirs-known,omitempty"`
|
||||
Merge string `json:"merge,omitempty"`
|
||||
Trees map[string]string `json:"trees,omitempty"`
|
||||
Root string `json:"root,omitempty"`
|
||||
// Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message,
|
||||
// of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path
|
||||
// relative to it (novox/hq ADR 0465 clause 1). Older writers send neither.
|
||||
Manifests map[string]string `json:"manifests,omitempty"`
|
||||
Locks map[string]map[string]string `json:"locks,omitempty"`
|
||||
// Paths are, of each module directory whose manifest names paths on every artifact, each named path's git object
|
||||
// id at the same commit as its manifest, by path relative to the directory (novox/hq ADR 0499 clause 3). The
|
||||
// indexer names the paths; a message without them gives such a module none. Older writers send none.
|
||||
Paths map[string]map[string]string `json:"paths,omitempty"`
|
||||
// Packages are, of each module directory, the mesh packages its lock file names (MeshPackagesOf; novox/hq ADR 0573
|
||||
// §2): the indexer keeps them in the module's inputs and takes each one's latest published version into its
|
||||
// fingerprint. Absent for a directory that uses none, and from an older writer.
|
||||
Packages map[string][]string `json:"packages,omitempty"`
|
||||
Act string `json:"act,omitempty"`
|
||||
Words string `json:"words,omitempty"`
|
||||
Open []OpenPull `json:"open,omitempty"`
|
||||
// RetiredPlacements is the placement files a bridge before ADR 0567 carried (ADR 0549 §2, ADR 0559 §2): placement is
|
||||
// the seat mesh-placement's state now and no repository holds it (ADR 0567 §11), so a message carrying them is
|
||||
// refused, loudly, rather than read with them dropped.
|
||||
RetiredPlacements json.RawMessage `json:"placements,omitempty"`
|
||||
RetiredRead bool `json:"placements-read,omitempty"`
|
||||
// Predefined is the catalogue's predefined.json, exact, at the merge commit of a merged message (novox/hq ADR 0565
|
||||
// §2): what the indexer keeps as its state `predefined`. PredefinedRead says the bridge read the merge and it holds
|
||||
// no predefined.json. Carried on a merged message only; an older bridge sends neither, and the indexer keeps what it
|
||||
// held.
|
||||
Predefined json.RawMessage `json:"predefined,omitempty"`
|
||||
PredefinedRead bool `json:"predefined-read,omitempty"`
|
||||
// Group is the delivery group a merged message's pull request was merged in, and Members every approved member of
|
||||
// that group with its approved head, this one included (novox/hq ADR 0633 §4): the forge bridge sets both from the
|
||||
// gate's merges state, and the indexer gathers the group's merges until the group's end (ForgeGroupEnd). A merge
|
||||
// with no group carries neither, and an older bridge sends neither; an older reader ignores both.
|
||||
Group string `json:"group,omitempty"`
|
||||
Members []delivery.Head `json:"members,omitempty"`
|
||||
}
|
||||
|
||||
// checkGroup refuses a group on anything but a merged message, a group without its members or members without a
|
||||
// group, a member that is not a repository's pull request at a head, a member named twice, and members that leave out
|
||||
// the message's own pull request.
|
||||
func (f Forge) checkGroup() error {
|
||||
if f.Group == "" && len(f.Members) == 0 {
|
||||
return nil
|
||||
}
|
||||
if f.Kind != ForgeMerged {
|
||||
return fmt.Errorf("delivery: only a merged message names its delivery group, not a %s message", f.Kind)
|
||||
}
|
||||
if f.Group == "" || len(f.Members) == 0 {
|
||||
return fmt.Errorf("delivery: a merged message names its group and the group's members together, or neither")
|
||||
}
|
||||
self := false
|
||||
seen := map[string]bool{}
|
||||
for _, m := range f.Members {
|
||||
if err := checkRepository(m.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Number <= 0 || m.Head == "" {
|
||||
return fmt.Errorf("delivery: the member %s of the group %q needs its pull request number and head", m.Repository, f.Group)
|
||||
}
|
||||
k := fmt.Sprintf("%s#%d", m.Repository, m.Number)
|
||||
if seen[k] {
|
||||
return fmt.Errorf("delivery: the group %q names its member %s twice", f.Group, k)
|
||||
}
|
||||
seen[k] = true
|
||||
self = self || (delivery.SameRepository(m.Repository, f.Repository) && m.Number == f.Number)
|
||||
}
|
||||
if !self {
|
||||
return fmt.Errorf("delivery: the members of the group %q leave out the merged pull request %s#%d", f.Group, f.Repository, f.Number)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkPredefined refuses the predefined list on anything but a merged message, a list that is not one JSON object, and
|
||||
// a list said both present and absent.
|
||||
func (f Forge) checkPredefined() error {
|
||||
if len(f.Predefined) == 0 && !f.PredefinedRead {
|
||||
return nil
|
||||
}
|
||||
if f.Kind != ForgeMerged {
|
||||
return fmt.Errorf("delivery: only a merged message carries the predefined list, not a %s message", f.Kind)
|
||||
}
|
||||
if len(f.Predefined) > 0 && f.PredefinedRead {
|
||||
return fmt.Errorf("delivery: a merged message carries the predefined list and says the merge holds none (predefined-read)")
|
||||
}
|
||||
var list map[string]json.RawMessage
|
||||
if len(f.Predefined) > 0 && json.Unmarshal(f.Predefined, &list) != nil {
|
||||
return fmt.Errorf("delivery: the predefined list a merged message carries is not one JSON object")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f Forge) check() error {
|
||||
if err := checkForgeKind(f.Kind); err != nil {
|
||||
return err
|
||||
}
|
||||
if f.Kind == ForgeOpen {
|
||||
return f.checkOpen()
|
||||
}
|
||||
if len(f.Open) > 0 {
|
||||
return fmt.Errorf("delivery: only an open message names open pull requests")
|
||||
}
|
||||
if len(f.RetiredPlacements) > 0 || f.RetiredRead {
|
||||
return fmt.Errorf("delivery: a forge message carries placement files, which no repository holds since ADR 0567: " +
|
||||
"placement is the seat mesh-placement's state")
|
||||
}
|
||||
if err := f.checkPredefined(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := f.checkGroup(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkRepository(f.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
switch f.Kind {
|
||||
case ForgePull:
|
||||
if f.Number <= 0 {
|
||||
return fmt.Errorf("delivery: a pull message needs its pull request number")
|
||||
}
|
||||
if f.Head == "" {
|
||||
return fmt.Errorf("delivery: a pull message needs its head")
|
||||
}
|
||||
switch f.Action {
|
||||
case PullOpened, PullUpdated, PullClosed:
|
||||
default:
|
||||
return fmt.Errorf("delivery: the pull action %q is not opened, updated or closed", f.Action)
|
||||
}
|
||||
if len(f.Dirs) > 0 && !f.DirsKnown {
|
||||
return fmt.Errorf("delivery: a pull message with module directories must say it knows them (dirs-known)")
|
||||
}
|
||||
dirs := map[string]bool{}
|
||||
for _, d := range f.Dirs {
|
||||
dirs[d] = true
|
||||
}
|
||||
if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil {
|
||||
return err
|
||||
}
|
||||
case ForgeMerged:
|
||||
if len(f.Dirs) > 0 || f.DirsKnown {
|
||||
return fmt.Errorf("delivery: a merged message carries no module directories")
|
||||
}
|
||||
if f.Number <= 0 {
|
||||
return fmt.Errorf("delivery: a merged message needs its pull request number")
|
||||
}
|
||||
if f.Head == "" {
|
||||
return fmt.Errorf("delivery: a merged message needs its head")
|
||||
}
|
||||
if f.Merge == "" {
|
||||
return fmt.Errorf("delivery: a merged message needs its merge commit")
|
||||
}
|
||||
if f.Root == "" {
|
||||
return fmt.Errorf("delivery: a merged message needs the repository's root tree")
|
||||
}
|
||||
// No trees is a merge that leaves the repository with no module.json anywhere (a documents repository, or one
|
||||
// whose last module was removed): it is written all the same, so the indexer takes the merge in and drops the
|
||||
// repository's modules (novox/hq issue 1126). An older writer always sent at least one tree, so none is unambiguous;
|
||||
// such a message carries no manifests, locks or paths (checkInputs below: each names a directory of its trees).
|
||||
for dir, tree := range f.Trees {
|
||||
if tree == "" {
|
||||
return fmt.Errorf("delivery: a merged message's tree of the directory %q is empty", dir)
|
||||
}
|
||||
}
|
||||
if err := f.checkInputs(func(d string) bool { _, ok := f.Trees[d]; return ok }, "its trees"); err != nil {
|
||||
return err
|
||||
}
|
||||
case ForgeRefused:
|
||||
if len(f.Dirs) > 0 || f.DirsKnown {
|
||||
return fmt.Errorf("delivery: a refused message carries no module directories")
|
||||
}
|
||||
if len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 {
|
||||
return fmt.Errorf("delivery: a refused message carries no manifests, lock hashes or named paths")
|
||||
}
|
||||
if f.Act == "" {
|
||||
return fmt.Errorf("delivery: a refused message needs the act it answers")
|
||||
}
|
||||
if f.Words == "" {
|
||||
return fmt.Errorf("delivery: a refused message needs the forge's words")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkInputs refuses a manifest or a lock hash of a directory that known does not know (of is what it knows them
|
||||
// by), an empty manifest and an empty lock path or hash, naming the directory.
|
||||
func (f Forge) checkInputs(known func(dir string) bool, of string) error {
|
||||
for dir, m := range f.Manifests {
|
||||
if !known(dir) {
|
||||
return fmt.Errorf("delivery: the manifest of the directory %q is not of %s", dir, of)
|
||||
}
|
||||
if m == "" {
|
||||
return fmt.Errorf("delivery: the manifest of the directory %q is empty", dir)
|
||||
}
|
||||
}
|
||||
for dir, locks := range f.Locks {
|
||||
if !known(dir) {
|
||||
return fmt.Errorf("delivery: the lock hashes of the directory %q are not of %s", dir, of)
|
||||
}
|
||||
for path, hash := range locks {
|
||||
if path == "" || hash == "" {
|
||||
return fmt.Errorf("delivery: the lock %q of the directory %q has an empty path or hash", path, dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
for dir, paths := range f.Paths {
|
||||
if !known(dir) {
|
||||
return fmt.Errorf("delivery: the named paths of the directory %q are not of %s", dir, of)
|
||||
}
|
||||
for path, object := range paths {
|
||||
if path == "" || object == "" {
|
||||
return fmt.Errorf("delivery: the named path %q of the directory %q has an empty path or object", path, dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkOpen refuses an open message that names a repository, a branch, a head or anything of the other kinds, or
|
||||
// an open pull request that is not `<owner>/<name>` and a number.
|
||||
func (f Forge) checkOpen() error {
|
||||
if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown ||
|
||||
f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 || f.Root != "" || f.Act != "" || f.Words != "" || f.Group != "" || len(f.Members) > 0 {
|
||||
return fmt.Errorf("delivery: an open message carries only the open pull requests it names")
|
||||
}
|
||||
for _, p := range f.Open {
|
||||
if err := checkRepository(p.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
if p.Number <= 0 {
|
||||
return fmt.Errorf("delivery: the open pull request of %s has no number", p.Repository)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeForge is f as JSON at Version; it refuses a message missing what its kind needs.
|
||||
func EncodeForge(f Forge) ([]byte, error) {
|
||||
if err := f.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.V = Version
|
||||
return json.Marshal(f)
|
||||
}
|
||||
|
||||
// DecodeForge reads a forge message; it refuses an unknown version (ErrUnknownVersion) and what EncodeForge refuses.
|
||||
func DecodeForge(data []byte) (Forge, error) {
|
||||
var f Forge
|
||||
if err := decode(data, &f); err != nil {
|
||||
return Forge{}, err
|
||||
}
|
||||
if err := f.check(); err != nil {
|
||||
return Forge{}, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// SameForge says whether two forge messages are the same value.
|
||||
func SameForge(a, b Forge) bool { return reflect.DeepEqual(a, b) }
|
||||
|
||||
var nameOK = regexp.MustCompile(`^[-/_=a-zA-Z0-9]+$`)
|
||||
|
||||
// checkName refuses a name that is empty or holds a dot, a wildcard, whitespace or a character a bus key refuses
|
||||
// (mesh-sdk delivery's rule for a name).
|
||||
func checkName(kind, name string) error {
|
||||
if name == "" {
|
||||
return fmt.Errorf("delivery: the %s name is empty", kind)
|
||||
}
|
||||
if !nameOK.MatchString(name) {
|
||||
return fmt.Errorf("delivery: the %s name %q holds a dot, a wildcard, whitespace or a character a key refuses", kind, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// decode reads a value of Version, refusing another or none with delivery.ErrUnknownVersion, as mesh-sdk delivery's
|
||||
// values are read.
|
||||
func decode(data []byte, into any) error {
|
||||
var probe struct {
|
||||
V *int `json:"v"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &probe); err != nil {
|
||||
return fmt.Errorf("delivery: value is not JSON: %w", err)
|
||||
}
|
||||
if probe.V == nil {
|
||||
return fmt.Errorf("%w: no version in the value, this reader reads %d", delivery.ErrUnknownVersion, Version)
|
||||
}
|
||||
if *probe.V != Version {
|
||||
return fmt.Errorf("%w: the value is version %d, this reader reads %d", delivery.ErrUnknownVersion, *probe.V, Version)
|
||||
}
|
||||
if err := json.Unmarshal(data, into); err != nil {
|
||||
return fmt.Errorf("delivery: value does not fit its shape: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package forgemsg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const oddManifest = "{\n\t\"module\": \"a\",\n \"z\": 1, \"a\": [ \"x\",\"y\" ],\n \"why\": \"é — <b> & \\n\"\n}\n\n"
|
||||
|
||||
func sampleForges() []Forge {
|
||||
return []Forge{
|
||||
{Kind: ForgePull, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 36, Head: "aaaa", Action: "opened"},
|
||||
{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc",
|
||||
Trees: map[string]string{"modules/a": "t1"}, Root: "t0"},
|
||||
{Kind: ForgeRefused, Repository: "novox/mesh-sdk", Act: "novox/mesh-sdk.36.aaaa.view", Words: "403 forbidden"},
|
||||
{Kind: ForgeOpen, Open: []OpenPull{{Repository: "novox/mesh-sdk", Number: 36}, {Repository: "novox/mesh-control", Number: 7}}},
|
||||
// 1585/T1: a group member's merged message names its group and every approved member (ADR 0633 §4).
|
||||
{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 37, Head: "dddd", Merge: "eeee", Root: "t0",
|
||||
Group: "feat/take-in", Members: []delivery.Head{{Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}, {Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}}},
|
||||
}
|
||||
}
|
||||
|
||||
func mergedWithInputs() Forge {
|
||||
return Forge{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc", Root: "t0",
|
||||
Trees: map[string]string{"a": "t1", "b": "t2"},
|
||||
Manifests: map[string]string{"a": oddManifest},
|
||||
Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("ab", 32)}}}
|
||||
}
|
||||
|
||||
// TestForge holds the forge bridge's contract, moved whole from mesh-sdk go/delivery with its tests (1200/AC1, AC3,
|
||||
// AC4; 1279/AC5; 1442/AC1-AC4; 1585/T1).
|
||||
func TestForge(t *testing.T) {
|
||||
t.Run("events", func(t *testing.T) {
|
||||
ev, err := ForgeEvent("novox/mesh-sdk", ForgeMerged)
|
||||
if err != nil || ev != "forge.novox.mesh-sdk.merged" {
|
||||
t.Fatalf("forge event = %q, %v", ev, err)
|
||||
}
|
||||
if repo, kind, err := ParseForgeEvent(ev); err != nil || repo != "novox/mesh-sdk" || kind != ForgeMerged {
|
||||
t.Fatalf("parse forge event = %q %q %v", repo, kind, err)
|
||||
}
|
||||
for _, bad := range []struct{ repo, kind string }{
|
||||
{"", "pull"}, {"novox", "pull"}, {"a/b/c", "pull"}, {"/b", "pull"}, {"a/", "pull"},
|
||||
{"a.b/c", "pull"}, {"a/b c", "pull"}, {"a/b", "closed"}, {"a/b", ""}, {"a/b", ForgeOpen},
|
||||
} {
|
||||
if _, err := ForgeEvent(bad.repo, bad.kind); err == nil {
|
||||
t.Errorf("forge event %q %q accepted", bad.repo, bad.kind)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "forge.a.b", "forge.a.b.closed", "x.a.b.pull", "forge.a.b.pull.x", "forge..b.pull", ForgeGroupEndEvent} {
|
||||
if _, _, err := ParseForgeEvent(bad); err == nil {
|
||||
t.Errorf("parse forge event %q accepted", bad)
|
||||
}
|
||||
}
|
||||
if repo, kind, err := ParseForgeEvent(OpenEvent()); OpenEvent() != "forge.open" || err != nil || repo != "" || kind != ForgeOpen {
|
||||
t.Errorf("open event %q = %q %q %v", OpenEvent(), repo, kind, err)
|
||||
}
|
||||
})
|
||||
t.Run("round trip", func(t *testing.T) {
|
||||
fs := append(sampleForges(), mergedWithInputs())
|
||||
for _, f := range fs {
|
||||
data, err := EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", f.Kind, err)
|
||||
}
|
||||
var probe map[string]any
|
||||
if err := json.Unmarshal(data, &probe); err != nil || probe["v"] != float64(1) {
|
||||
t.Fatalf("%s: json %s does not carry v:1", f.Kind, data)
|
||||
}
|
||||
f.V = Version
|
||||
if got, err := DecodeForge(data); err != nil || !SameForge(got, f) {
|
||||
t.Errorf("forge %s round trip: %+v %v", f.Kind, got, err)
|
||||
}
|
||||
}
|
||||
// An empty list is an open message too: nothing is open.
|
||||
if data, err := EncodeForge(Forge{Kind: ForgeOpen, Open: []OpenPull{}}); err != nil {
|
||||
t.Errorf("empty open: %v", err)
|
||||
} else if f, err := DecodeForge(data); err != nil || f.Kind != ForgeOpen || len(f.Open) != 0 {
|
||||
t.Errorf("empty open round trip: %+v %v", f, err)
|
||||
}
|
||||
ge := ForgeGroupEnd{Group: "feat/take-in", Members: []GroupMerge{{Repository: "novox/mesh-sdk", Number: 37, Merge: "eeee"}}}
|
||||
data, err := EncodeForgeGroupEnd(ge)
|
||||
ge.V = Version
|
||||
if got, err2 := DecodeForgeGroupEnd(data); err != nil || err2 != nil || !SameForgeGroupEnd(got, ge) {
|
||||
t.Errorf("group end round trip: %+v %v %v", got, err, err2)
|
||||
}
|
||||
// An older bridge's message reads, and is written back byte for byte.
|
||||
old := `{"v":1,"kind":"merged","repository":"novox/mesh-sdk","number":36,"head":"aaaa","merge":"cccc","trees":{"a":"t1"},"root":"t0"}`
|
||||
if f, err := DecodeForge([]byte(old)); err != nil || f.Manifests != nil {
|
||||
t.Fatalf("%+v, %v", f, err)
|
||||
} else if data, err := EncodeForge(f); err != nil || string(data) != old {
|
||||
t.Errorf("%s, %v, want exactly %s", data, err, old)
|
||||
}
|
||||
// A merge leaving the repository with no module.json is a message too (issue 1126).
|
||||
f := sampleForges()[1]
|
||||
f.Trees = nil
|
||||
if data, err := EncodeForge(f); err != nil {
|
||||
t.Errorf("a merged message with no trees is refused: %v", err)
|
||||
} else if got, err := DecodeForge(data); err != nil || len(got.Trees) != 0 {
|
||||
t.Errorf("a merged message with no trees comes back as %+v, %v", got, err)
|
||||
}
|
||||
for name, dec := range map[string]func([]byte) error{
|
||||
"forge": func(b []byte) error { _, err := DecodeForge(b); return err },
|
||||
"group end": func(b []byte) error { _, err := DecodeForgeGroupEnd(b); return err },
|
||||
} {
|
||||
for _, in := range []string{`{"group":"g"}`, `{"v":2,"group":"g"}`} {
|
||||
if err := dec([]byte(in)); !errors.Is(err, delivery.ErrUnknownVersion) {
|
||||
t.Errorf("%s decode of %s = %v, want delivery.ErrUnknownVersion", name, in, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("refused", func(t *testing.T) {
|
||||
at := func(i int, mut func(*Forge)) Forge { f := sampleForges()[i]; mut(&f); return f }
|
||||
in := func(mut func(*Forge)) Forge { f := mergedWithInputs(); mut(&f); return f }
|
||||
pullIn := func(mut func(*Forge)) Forge {
|
||||
f := Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened,
|
||||
Dirs: []string{"a"}, DirsKnown: true, Manifests: map[string]string{"a": oddManifest},
|
||||
Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("0", 64)}}}
|
||||
mut(&f)
|
||||
return f
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name, want string
|
||||
f Forge
|
||||
}{
|
||||
{"merged without merge commit", "merge", at(1, func(f *Forge) { f.Merge = "" })},
|
||||
{"merged without root", "root", at(1, func(f *Forge) { f.Root = "" })},
|
||||
{"merged with a manifest and no trees", "modules/a", at(1, func(f *Forge) { f.Trees = nil; f.Manifests = map[string]string{"modules/a": "{}"} })},
|
||||
{"merged with an empty tree", "modules/a", at(1, func(f *Forge) { f.Trees = map[string]string{"modules/a": ""} })},
|
||||
{"pull with an unknown action", "reopened", at(0, func(f *Forge) { f.Action = "reopened" })},
|
||||
{"pull without head", "head", at(0, func(f *Forge) { f.Head = "" })},
|
||||
{"refused without words", "words", at(2, func(f *Forge) { f.Words = "" })},
|
||||
{"refused without act", "act", at(2, func(f *Forge) { f.Act = "" })},
|
||||
{"unknown kind", "closed", Forge{Kind: "closed", Repository: "a/b"}},
|
||||
{"bad repository", "nope", Forge{Kind: ForgePull, Repository: "nope", Number: 1, Head: "h", Action: "opened"}},
|
||||
{"open with a repository", "only the open", Forge{Kind: ForgeOpen, Repository: "a/b", Open: []OpenPull{{Repository: "a/b", Number: 1}}}},
|
||||
{"open with a bad entry", "", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a", Number: 1}}}},
|
||||
{"open with a zero number", "no number", Forge{Kind: ForgeOpen, Open: []OpenPull{{Repository: "a/b"}}}},
|
||||
{"pull naming open pull requests", "only an open message", Forge{Kind: ForgePull, Repository: "a/b", Number: 1, Head: "h", Action: "updated", Open: []OpenPull{{Repository: "a/b", Number: 1}}}},
|
||||
{"manifest outside trees", `"zz"`, in(func(f *Forge) { f.Manifests["zz"] = "{}" })},
|
||||
{"locks outside trees", `"yy"`, in(func(f *Forge) { f.Locks["yy"] = map[string]string{"go.sum": "h"} })},
|
||||
{"empty manifest", `"b"`, in(func(f *Forge) { f.Manifests["b"] = "" })},
|
||||
{"empty lock hash", `"a"`, in(func(f *Forge) { f.Locks["a"]["go.sum"] = "" })},
|
||||
{"named paths outside trees", `"other"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"other": {"src": "o"}} })},
|
||||
{"a named path with no object", `"src"`, in(func(f *Forge) { f.Paths = map[string]map[string]string{"a": {"src": ""}} })},
|
||||
{"placement files", "mesh-placement", in(func(f *Forge) { f.RetiredPlacements = []byte(`{"anchor":"{}"}`) })},
|
||||
{"placements read", "mesh-placement", in(func(f *Forge) { f.RetiredRead = true })},
|
||||
{"predefined not an object", "one JSON object", in(func(f *Forge) { f.Predefined = []byte(`["domain"]`) })},
|
||||
{"predefined both", "says the merge holds none", in(func(f *Forge) { f.Predefined = []byte(`{}`); f.PredefinedRead = true })},
|
||||
{"pull with the predefined list", "only a merged message", Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: PullOpened, PredefinedRead: true}},
|
||||
{"pull manifest outside dirs", `"b"`, pullIn(func(f *Forge) { f.Manifests["b"] = "{}" })},
|
||||
{"pull locks outside dirs", `"c"`, pullIn(func(f *Forge) { f.Locks["c"] = map[string]string{"go.sum": "h"} })},
|
||||
{"refused with manifests", "no manifests", Forge{Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Manifests: map[string]string{"a": "{}"}}},
|
||||
{"open with locks", "only the open", Forge{Kind: ForgeOpen, Locks: map[string]map[string]string{"a": {"go.sum": "h"}}}},
|
||||
// 1585/T1: a group only on a merged message, with its members, the message's own pull request among them.
|
||||
{"pull with a group", "only a merged message names its delivery group", at(0, func(f *Forge) {
|
||||
f.Group, f.Members = "g", []delivery.Head{{Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa"}}
|
||||
})},
|
||||
{"merged with a group and no members", "together", at(4, func(f *Forge) { f.Members = nil })},
|
||||
{"merged with members and no group", "together", at(4, func(f *Forge) { f.Group = "" })},
|
||||
{"merged with members leaving itself out", "leave out", at(4, func(f *Forge) { f.Members = f.Members[1:] })},
|
||||
{"merged with a member twice", "twice", at(4, func(f *Forge) { f.Members = append(f.Members, f.Members[0]) })},
|
||||
{"open with a group", "only the open pull requests", Forge{Kind: ForgeOpen, Group: "g"}},
|
||||
} {
|
||||
if _, err := EncodeForge(c.f); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want)
|
||||
}
|
||||
}
|
||||
// A member's repository in another case still names the merged pull request (10c91fba).
|
||||
f := sampleForges()[4]
|
||||
f.Members[0].Repository = "Novox/Mesh-SDK"
|
||||
if _, err := EncodeForge(f); err != nil {
|
||||
t.Errorf("a member in another case: %v", err)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name, want string
|
||||
e ForgeGroupEnd
|
||||
}{
|
||||
{"group end without members", "no member", ForgeGroupEnd{Group: "g"}},
|
||||
{"group end without a group", "names its group", ForgeGroupEnd{Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}}}},
|
||||
{"group end with a member without its merge", "merge commit", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1}}}},
|
||||
{"group end with a member twice", "twice", ForgeGroupEnd{Group: "g", Members: []GroupMerge{{Repository: "a/b", Number: 1, Merge: "m"}, {Repository: "a/b", Number: 1, Merge: "m"}}}},
|
||||
} {
|
||||
if _, err := EncodeForgeGroupEnd(c.e); err == nil || !strings.Contains(err.Error(), c.want) {
|
||||
t.Errorf("%s: %v, want a refusal naming %q", c.name, err, c.want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
module git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
|
||||
@@ -0,0 +1,14 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
@@ -0,0 +1,78 @@
|
||||
package forgemsg
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// ForgeGroupEndEvent is the name of a delivery group's end on the forge bridge's stream (novox/hq ADR 0633 §4): the
|
||||
// bridge publishes it once every approved member of the group is merged, after the last member's merged message. It
|
||||
// has three parts, so no repository's forge event (`forge.<owner>.<name>.<kind>`) is ever this name.
|
||||
const ForgeGroupEndEvent = "forge.group.merged"
|
||||
|
||||
// GroupMerge is one merged member of a delivery group: the repository as `<owner>/<name>`, its pull request and the
|
||||
// merge commit the forge made of it.
|
||||
type GroupMerge struct {
|
||||
Repository string `json:"repository"`
|
||||
Number int `json:"number"`
|
||||
Merge string `json:"merge"`
|
||||
}
|
||||
|
||||
// ForgeGroupEnd is the body of ForgeGroupEndEvent: the group and every member merged in it, with its merge commit.
|
||||
// The indexer takes the members it holds in as one source revision once it hears it (ADR 0633 §4).
|
||||
type ForgeGroupEnd struct {
|
||||
V int `json:"v"`
|
||||
Group string `json:"group"`
|
||||
Members []GroupMerge `json:"members"`
|
||||
}
|
||||
|
||||
func (e ForgeGroupEnd) check() error {
|
||||
if e.Group == "" {
|
||||
return fmt.Errorf("delivery: a group's end names its group")
|
||||
}
|
||||
if len(e.Members) == 0 {
|
||||
return fmt.Errorf("delivery: the end of the group %q names no member merged in it", e.Group)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, m := range e.Members {
|
||||
if err := checkRepository(m.Repository); err != nil {
|
||||
return err
|
||||
}
|
||||
if m.Number <= 0 || m.Merge == "" {
|
||||
return fmt.Errorf("delivery: the member %s of the group %q's end needs its pull request number and merge commit", m.Repository, e.Group)
|
||||
}
|
||||
k := fmt.Sprintf("%s#%d", m.Repository, m.Number)
|
||||
if seen[k] {
|
||||
return fmt.Errorf("delivery: the end of the group %q names its member %s twice", e.Group, k)
|
||||
}
|
||||
seen[k] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EncodeForgeGroupEnd is e as JSON at Version; it refuses an end with no group or no member, and a member without its
|
||||
// pull request or merge commit.
|
||||
func EncodeForgeGroupEnd(e ForgeGroupEnd) ([]byte, error) {
|
||||
if err := e.check(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e.V = Version
|
||||
return json.Marshal(e)
|
||||
}
|
||||
|
||||
// DecodeForgeGroupEnd reads a group's end; it refuses an unknown version (ErrUnknownVersion) and what
|
||||
// EncodeForgeGroupEnd refuses.
|
||||
func DecodeForgeGroupEnd(data []byte) (ForgeGroupEnd, error) {
|
||||
var e ForgeGroupEnd
|
||||
if err := decode(data, &e); err != nil {
|
||||
return ForgeGroupEnd{}, err
|
||||
}
|
||||
if err := e.check(); err != nil {
|
||||
return ForgeGroupEnd{}, err
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// SameForgeGroupEnd says whether two group ends are the same value.
|
||||
func SameForgeGroupEnd(a, b ForgeGroupEnd) bool { return reflect.DeepEqual(a, b) }
|
||||
@@ -3,12 +3,13 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
@@ -20,3 +21,5 @@ require (
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ./forgemsg
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -145,11 +146,11 @@ func MergeActs(value []byte) ([]Act, error) {
|
||||
// RefusedMessage is the refused message the bridge writes on FORGE when the forge refuses an act: the act, the pull
|
||||
// request and head it was on, and the forge's own words.
|
||||
func RefusedMessage(a Act, words string) (Message, error) {
|
||||
event, err := delivery.ForgeEvent(a.Repository, delivery.ForgeRefused)
|
||||
event, err := forgemsg.ForgeEvent(a.Repository, forgemsg.ForgeRefused)
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeRefused, Repository: a.Repository, Number: a.Number,
|
||||
body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeRefused, Repository: a.Repository, Number: a.Number,
|
||||
Head: a.Head, Act: a.Name, Words: words})
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -27,7 +28,7 @@ func owedValue(t *testing.T, repository string, number int, act string, o delive
|
||||
return key, b
|
||||
}
|
||||
|
||||
func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1
|
||||
func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1, K1
|
||||
key, v := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate",
|
||||
delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "pending", Text: "waiting", Target: "https://forge.example/novox/mesh-delivery/pulls/12"})
|
||||
a, err := OwedAct(key, v)
|
||||
@@ -42,6 +43,19 @@ func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1
|
||||
if a.Repository != "novox/mesh-delivery" || a.Number != 12 || a.Head != headA || a.Name != "status=mesh/merge-gate" {
|
||||
t.Errorf("act = %+v", a)
|
||||
}
|
||||
// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no
|
||||
// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge
|
||||
// performs no mesh/check act.
|
||||
ck, cv := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"})
|
||||
if a, err := OwedAct(ck, cv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
|
||||
t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err)
|
||||
}
|
||||
// Nor through the gate's own owed bucket: the context is refused whichever key carries it.
|
||||
gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check",
|
||||
delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"})
|
||||
if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
|
||||
t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
// checkerKV is the status the checker owes, as it writes it (mesh-sdk go/delivery CheckerOwedKey, EncodeCheckerOwed).
|
||||
@@ -58,23 +72,6 @@ func checkerKV(t *testing.T, o delivery.Owed) (string, []byte) {
|
||||
return key, v
|
||||
}
|
||||
|
||||
// R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no
|
||||
// caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge
|
||||
// performs no mesh/check act.
|
||||
func TestOwedMeshCheckIsNoToolCall(t *testing.T) { // K1
|
||||
key, v := checkerKV(t, delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"})
|
||||
a, err := OwedAct(key, v)
|
||||
if err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
|
||||
t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err)
|
||||
}
|
||||
// Nor through the gate's own owed bucket: the context is refused whichever key carries it.
|
||||
gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check",
|
||||
delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"})
|
||||
if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") {
|
||||
t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedViewNoteAndCommentCalls(t *testing.T) { // 1341/AC2
|
||||
cases := []struct {
|
||||
act string
|
||||
@@ -111,18 +108,12 @@ func TestOwedActWithoutAPullRequestNeedsOneOnlyWhereTheToolDoes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedActOfADottedRepositoryIsUnescaped(t *testing.T) {
|
||||
key, v := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"})
|
||||
a, err := OwedAct(key, v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" {
|
||||
t.Errorf("act = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedActThatDisagreesWithItsKeyIsRefused(t *testing.T) {
|
||||
// A dotted repository's key is unescaped, and its act names the repository whole.
|
||||
dk, dv := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"})
|
||||
if a, err := OwedAct(dk, dv); err != nil || a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" {
|
||||
t.Errorf("a dotted repository's act = %+v, %v", a, err)
|
||||
}
|
||||
key, _ := owedValue(t, "novox/mesh-delivery", 12, "view", delivery.Owed{Act: delivery.OwedView, Text: "v"})
|
||||
v, _ := delivery.EncodeOwed(delivery.Owed{Act: delivery.OwedComment, Text: "v"})
|
||||
if _, err := OwedAct(key, v); err == nil {
|
||||
@@ -166,13 +157,11 @@ func TestMergeActsHandOnlyTheMemberHandedWithEveryApprovedHead(t *testing.T) { /
|
||||
if acts[0].Name != "merge" || acts[0].Number != 12 || acts[0].Head != headA {
|
||||
t.Errorf("act = %+v", acts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeActsOfAMergeWithoutApprovedHeadsAreRefused(t *testing.T) {
|
||||
v := mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{
|
||||
// A merge with no approved heads is refused: the forge's tool would refuse it with nothing to say which.
|
||||
v = mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, State: delivery.MergeMerging}}})
|
||||
if _, err := MergeActs(v); err == nil {
|
||||
t.Error("a merge with no approved heads was taken: the forge's tool would refuse it with nothing to say which")
|
||||
t.Error("a merge with no approved heads was taken")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -185,13 +174,13 @@ func TestRefusedMessageNamesTheActAndTheForgesWords(t *testing.T) { // 1341/AC3,
|
||||
if m.Event != "forge.novox.mesh-delivery.refused" {
|
||||
t.Errorf("event = %q", m.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(m.Data)
|
||||
f, err := forgemsg.DecodeForge(m.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA,
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA,
|
||||
Act: "merge", Words: "novox/mesh-delivery#12 is not merged: its head moved"}
|
||||
if !delivery.SameForge(f, want) {
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// ErrUnreadable is wrapped by every error that says an event cannot be turned into a message; it is never
|
||||
@@ -110,11 +110,11 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
|
||||
var action string
|
||||
switch subject {
|
||||
case UpdatedSubject:
|
||||
action = delivery.PullUpdated
|
||||
action = forgemsg.PullUpdated
|
||||
case ClosedSubject:
|
||||
action = delivery.PullClosed
|
||||
action = forgemsg.PullClosed
|
||||
case MergedSubject:
|
||||
action = delivery.PullClosed // unless it merged into the trunk, below
|
||||
action = forgemsg.PullClosed // unless it merged into the trunk, below
|
||||
default:
|
||||
return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject)
|
||||
}
|
||||
@@ -136,29 +136,29 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
|
||||
return merged(subject, p, trees, files, kept)
|
||||
}
|
||||
said := ""
|
||||
if action == delivery.PullUpdated && p.Base != "" && p.Base != Trunk {
|
||||
if action == forgemsg.PullUpdated && p.Base != "" && p.Base != Trunk {
|
||||
// The checker judges pull requests into main only, and source.main names main's head only (novox/hq ADR 0484 §11): a
|
||||
// pull request into another base is no candidate, as it is none at the start (OpenPulls) and at its merge. It is
|
||||
// written closed, so a candidate an older head made is taken out, and said (issue 1422).
|
||||
action = delivery.PullClosed
|
||||
action = forgemsg.PullClosed
|
||||
said = fmt.Sprintf("%s/%s#%d at %s is into %s, not %s: written closed, so it is no candidate; the checker judges pull requests into %s only (novox/hq ADR 0484 §11)",
|
||||
p.Owner, p.Repo, p.Number, p.HeadSHA[:12], p.Base, Trunk, Trunk)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: action}
|
||||
// The directories are the forge's word only when it said it could read them (module_dirs_said).
|
||||
if action == delivery.PullUpdated && p.ModuleDirsSaid {
|
||||
if action == forgemsg.PullUpdated && p.ModuleDirsSaid {
|
||||
f.Dirs, f.DirsKnown = p.ModuleDirs, true
|
||||
var err error
|
||||
if f.Manifests, f.Locks, _, err = inputsRead(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
|
||||
}
|
||||
}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
body, err := delivery.EncodeForge(f)
|
||||
body, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
@@ -313,11 +313,11 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
|
||||
if len(locks) == 0 {
|
||||
locks = nil
|
||||
}
|
||||
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
|
||||
event, err := forgemsg.ForgeEvent(repository, forgemsg.ForgeMerged)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
f := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
|
||||
if repository == CatalogRepository {
|
||||
// The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config.
|
||||
@@ -331,7 +331,7 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
|
||||
f.Predefined = json.RawMessage(raw)
|
||||
}
|
||||
}
|
||||
body, err := delivery.EncodeForge(f)
|
||||
body, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
@@ -373,12 +373,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
|
||||
skipped = append(skipped, name+": it names no owner, repo, number, branch or 40-hex head")
|
||||
continue
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: delivery.PullUpdated}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
f := forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: forgemsg.PullUpdated}
|
||||
event, err := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
|
||||
var body []byte
|
||||
if err == nil {
|
||||
body, err = delivery.EncodeForge(f)
|
||||
body, err = forgemsg.EncodeForge(f)
|
||||
}
|
||||
if err != nil {
|
||||
skipped = append(skipped, name+": "+err.Error())
|
||||
@@ -392,12 +392,12 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
|
||||
// Track says what a message the bridge wrote changes in the head it keeps for each pull request (the bucket heads,
|
||||
// novox/hq ADR 0443 §5): the key `<owner>.<name>.<number>`, and the head to keep under it, or "" to take the key away
|
||||
// (a closed pull request, a merged one). ok is false for a message that keeps no head.
|
||||
func Track(f delivery.Forge) (key, head string, ok bool) {
|
||||
func Track(f forgemsg.Forge) (key, head string, ok bool) {
|
||||
key = strings.ReplaceAll(f.Repository, "/", ".") + "." + strconv.Itoa(f.Number)
|
||||
switch {
|
||||
case f.Kind == delivery.ForgePull && f.Action != delivery.PullClosed:
|
||||
case f.Kind == forgemsg.ForgePull && f.Action != forgemsg.PullClosed:
|
||||
return key, f.Head, true
|
||||
case f.Kind == delivery.ForgePull || f.Kind == delivery.ForgeMerged:
|
||||
case f.Kind == forgemsg.ForgePull || f.Kind == forgemsg.ForgeMerged:
|
||||
return key, "", true
|
||||
}
|
||||
return "", "", false
|
||||
@@ -413,22 +413,22 @@ func OpenMessage(answer []byte) (Message, error) {
|
||||
if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil {
|
||||
return Message{}, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err)
|
||||
}
|
||||
open := []delivery.OpenPull{}
|
||||
open := []forgemsg.OpenPull{}
|
||||
for _, p := range *a.Pulls {
|
||||
if p.Base != Trunk || p.Owner == "" || p.Repo == "" || p.Number <= 0 {
|
||||
continue
|
||||
}
|
||||
open = append(open, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
|
||||
open = append(open, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
|
||||
}
|
||||
slices.SortFunc(open, func(a, b delivery.OpenPull) int {
|
||||
slices.SortFunc(open, func(a, b forgemsg.OpenPull) int {
|
||||
if c := strings.Compare(a.Repository, b.Repository); c != 0 {
|
||||
return c
|
||||
}
|
||||
return a.Number - b.Number
|
||||
})
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeOpen, Open: open})
|
||||
body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeOpen, Open: open})
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("the open pull requests cannot be said: %w", err)
|
||||
}
|
||||
return Message{Event: delivery.OpenEvent(), Data: body}, nil
|
||||
return Message{Event: forgemsg.OpenEvent(), Data: body}, nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -54,13 +55,13 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion;
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" {
|
||||
t.Errorf("event = %q", msg.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
}
|
||||
@@ -70,7 +71,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ := delivery.DecodeForge(msg.Data)
|
||||
f, _ := forgemsg.DecodeForge(msg.Data)
|
||||
if f.DirsKnown || len(f.Dirs) != 0 {
|
||||
t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
@@ -79,7 +80,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ = delivery.DecodeForge(msg.Data)
|
||||
f, _ = forgemsg.DecodeForge(msg.Data)
|
||||
if !f.DirsKnown || len(f.Dirs) != 0 {
|
||||
t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
@@ -88,7 +89,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ = delivery.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 {
|
||||
if f, _ = forgemsg.DecodeForge(msg.Data); f.DirsKnown || len(f.Dirs) != 0 {
|
||||
t.Errorf("dirs %v known %v without module_dirs_said", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
}
|
||||
@@ -101,13 +102,13 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" {
|
||||
t.Errorf("event = %q", msg.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullClosed}
|
||||
if !delivery.SameForge(f, want) {
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: forgemsg.PullClosed}
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
// Issue 1422, ADR 0484 §11: a head pushed to a pull request into another base is written closed, read nothing for, and said;
|
||||
@@ -116,12 +117,12 @@ func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
|
||||
if f, err = forgemsg.DecodeForge(msg.Data); err != nil || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "is into nox-mesh, not main") {
|
||||
t.Errorf("an update into nox-mesh is %+v (%v), said %q; want it closed and said", f, err, msg.Said)
|
||||
}
|
||||
for _, base := range []any{"main", nil} {
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"base": base}), nil, anyFiles)
|
||||
if f, _ = delivery.DecodeForge(msg.Data); err != nil || f.Action != delivery.PullUpdated || msg.Said != "" {
|
||||
if f, _ = forgemsg.DecodeForge(msg.Data); err != nil || f.Action != forgemsg.PullUpdated || msg.Said != "" {
|
||||
t.Errorf("an update with base %v is %+v (%v), said %q; want it updated", base, f, err, msg.Said)
|
||||
}
|
||||
}
|
||||
@@ -170,12 +171,18 @@ func mustDrop(body []byte, key string) []byte {
|
||||
return b
|
||||
}
|
||||
|
||||
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235
|
||||
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236
|
||||
for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} {
|
||||
if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) {
|
||||
t.Errorf("%s: err = %v, want ErrNotHandled", s, err)
|
||||
}
|
||||
}
|
||||
// 1236: a merged event without a full merge commit is unreadable, and the forge is not asked.
|
||||
var calls []call
|
||||
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
|
||||
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
|
||||
t.Errorf("merged without a merge commit: err = %v calls = %v", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
@@ -222,13 +229,13 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
|
||||
if msg.Event != "forge.novox.mesh-delivery.merged" {
|
||||
t.Errorf("event = %q", msg.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
if !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
if len(calls) != 1 || calls[0] != (call{"novox", "mesh-delivery", merge}) {
|
||||
@@ -242,13 +249,13 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullClosed}
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" || !delivery.SameForge(f, want) {
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: forgemsg.PullClosed}
|
||||
if msg.Event != "forge.novox.mesh-delivery.pull" || !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("event %q message %+v, want a closed pull message", msg.Event, f)
|
||||
}
|
||||
if len(calls) != 0 {
|
||||
@@ -277,22 +284,14 @@ func TestNoModuleJSONAnywhereWritesAMergedMessageWithNoTrees(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
want := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Merge: merge, Root: "r0"}
|
||||
if err != nil || msg.Event != "forge.novox.mesh-delivery.merged" || !delivery.SameForge(f, want) || !strings.Contains(msg.Said, "no module.json") {
|
||||
if err != nil || msg.Event != "forge.novox.mesh-delivery.merged" || !forgemsg.SameForge(f, want) || !strings.Contains(msg.Said, "no module.json") {
|
||||
t.Errorf("event %q message %+v said %q, %v; want a merged message with no trees", msg.Event, f, msg.Said, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236
|
||||
var calls []call
|
||||
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
|
||||
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
|
||||
t.Errorf("err = %v calls = %v", err, calls)
|
||||
}
|
||||
}
|
||||
|
||||
// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds each entry
|
||||
// (owner, repo, number, base, head, head_sha, html_url, required, statuses, checks, labels) inside {pulls}.
|
||||
func openAnswer(pulls ...map[string]any) []byte {
|
||||
@@ -311,12 +310,12 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/
|
||||
if err != nil || len(skipped) != 0 || len(msgs) != 2 {
|
||||
t.Fatalf("msgs %d skipped %v err %v", len(msgs), skipped, err)
|
||||
}
|
||||
for i, want := range []delivery.Forge{
|
||||
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated},
|
||||
{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated},
|
||||
for i, want := range []forgemsg.Forge{
|
||||
{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: forgemsg.PullUpdated},
|
||||
{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated},
|
||||
} {
|
||||
f, err := delivery.DecodeForge(msgs[i].Data)
|
||||
if err != nil || !delivery.SameForge(f, want) {
|
||||
f, err := forgemsg.DecodeForge(msgs[i].Data)
|
||||
if err != nil || !forgemsg.SameForge(f, want) {
|
||||
t.Errorf("message %d = %+v err %v, want %+v", i, f, err, want)
|
||||
}
|
||||
if f.DirsKnown {
|
||||
@@ -328,14 +327,11 @@ func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenPullIntoAnotherBranchGetsNoMessage(t *testing.T) { // 1237/AC4
|
||||
msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head)))
|
||||
if err != nil || len(msgs) != 0 || len(skipped) != 0 {
|
||||
t.Errorf("msgs %d skipped %v err %v, want none", len(msgs), skipped, err)
|
||||
func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237, 1237/AC4
|
||||
// 1237/AC4: an open pull request into another branch gets no message.
|
||||
if msgs, skipped, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "feat/y", "feat/x", head))); err != nil || len(msgs) != 0 || len(skipped) != 0 {
|
||||
t.Errorf("into another branch: msgs %d skipped %v err %v, want none", len(msgs), skipped, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237
|
||||
msgs, skipped, err := OpenPulls(openAnswer(openEntry("a", 1, "main", "b", "short"), openEntry("mesh-delivery", 12, "main", "feat/x", head)))
|
||||
if err != nil || len(msgs) != 1 || len(skipped) != 1 || !strings.Contains(skipped[0], "novox/a#1") {
|
||||
t.Errorf("msgs %d skipped %v err %v", len(msgs), skipped, err)
|
||||
@@ -355,15 +351,15 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6
|
||||
if err != nil || m.Event != "forge.open" {
|
||||
t.Fatalf("message %+v err %v", m, err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(m.Data)
|
||||
f, err := forgemsg.DecodeForge(m.Data)
|
||||
// Every one into the trunk, whether its head reads or not (it is open); none into another branch; in a fixed order.
|
||||
want := []delivery.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
if err != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] {
|
||||
want := []forgemsg.OpenPull{{Repository: "novox/a", Number: 1}, {Repository: "novox/mesh-catalog", Number: 300}, {Repository: "novox/mesh-delivery", Number: 12}}
|
||||
if err != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 3 || f.Open[0] != want[0] || f.Open[1] != want[1] || f.Open[2] != want[2] {
|
||||
t.Errorf("open message = %+v err %v, want %v", f, err, want)
|
||||
}
|
||||
// Nothing open is a message too: the indexer then drops every candidate.
|
||||
m, err = OpenMessage([]byte(`{"pulls":[]}`))
|
||||
if f, derr := delivery.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != delivery.ForgeOpen || len(f.Open) != 0 {
|
||||
if f, derr := forgemsg.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 0 {
|
||||
t.Errorf("empty answer: %+v %v %v", f, err, derr)
|
||||
}
|
||||
for _, bad := range []string{`{"pulls":null}`, `{}`, `x`} {
|
||||
@@ -374,24 +370,24 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6
|
||||
}
|
||||
|
||||
func TestTrackSaysWhichHeadOfWhichPullRequestWasWritten(t *testing.T) { // 1279/AC6
|
||||
pullMsg := func(action string) delivery.Forge {
|
||||
return delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action}
|
||||
pullMsg := func(action string) forgemsg.Forge {
|
||||
return forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action}
|
||||
}
|
||||
for _, a := range []string{delivery.PullOpened, delivery.PullUpdated} {
|
||||
for _, a := range []string{forgemsg.PullOpened, forgemsg.PullUpdated} {
|
||||
if key, h, ok := Track(pullMsg(a)); key != "novox.mesh-delivery.12" || h != head || !ok {
|
||||
t.Errorf("%s: Track = %q %q %v, want the head kept under a bucket key", a, key, h, ok)
|
||||
}
|
||||
}
|
||||
// A closed pull request and a merged one are no longer kept: their key goes.
|
||||
if key, h, ok := Track(pullMsg(delivery.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok {
|
||||
if key, h, ok := Track(pullMsg(forgemsg.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok {
|
||||
t.Errorf("closed: Track = %q %q %v", key, h, ok)
|
||||
}
|
||||
merged := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}
|
||||
merged := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}
|
||||
if key, h, ok := Track(merged); key != "novox.mesh-delivery.12" || h != "" || !ok {
|
||||
t.Errorf("merged: Track = %q %q %v", key, h, ok)
|
||||
}
|
||||
// The open message and the rest hold no head.
|
||||
if _, _, ok := Track(delivery.Forge{Kind: delivery.ForgeOpen}); ok {
|
||||
if _, _, ok := Track(forgemsg.Forge{Kind: forgemsg.ForgeOpen}); ok {
|
||||
t.Error("an open message was tracked")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// Caller calls a module's tool by its manifest key (`<module>.<tool>`) with a JSON body and returns the tool's result:
|
||||
// the bridge's process calls through the bus as its own principal, its tools bundle through the runtime (stdio.Ask).
|
||||
type Caller func(ctx context.Context, key string, body any) (json.RawMessage, error)
|
||||
|
||||
// CommitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner,
|
||||
// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4).
|
||||
const CommitTreesTool = "gitea.gitea_commit_trees"
|
||||
|
||||
// TreesVia asks the forge for a commit's trees through call.
|
||||
func TreesVia(ctx context.Context, call Caller) Trees {
|
||||
return func(owner, repo, sha string) (string, map[string]string, error) {
|
||||
raw, err := call(ctx, CommitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var a struct {
|
||||
Root string `json:"root"`
|
||||
Trees map[string]string `json:"trees"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", CommitTreesTool, err)
|
||||
}
|
||||
if a.Root == "" {
|
||||
return "", nil, fmt.Errorf("the answer of %s names no root tree", CommitTreesTool)
|
||||
}
|
||||
return a.Root, a.Trees, nil
|
||||
}
|
||||
}
|
||||
|
||||
// GetFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
|
||||
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
|
||||
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
|
||||
const GetFileTool = "gitea.gitea_get_file"
|
||||
|
||||
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
|
||||
// recognises it as /: 404 /).
|
||||
var notFound = regexp.MustCompile(`: 404 `)
|
||||
|
||||
// FilesVia asks the forge for a file through call. A file the forge says is not there is ErrNotFound.
|
||||
func FilesVia(ctx context.Context, call Caller) Files {
|
||||
return func(owner, repo, path, ref string) ([]byte, error) {
|
||||
raw, err := call(ctx, GetFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
|
||||
if err != nil {
|
||||
if notFound.MatchString(err.Error()) {
|
||||
return nil, fmt.Errorf("%w: %v", ErrNotFound, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", GetFileTool, err)
|
||||
}
|
||||
return []byte(*a.File.Content), nil
|
||||
}
|
||||
}
|
||||
|
||||
// MergedMessage is the merged message of a merge into the trunk, asked again: built as the bridge builds the one it writes
|
||||
// on FORGE (it is that event's body through ConvertKept), from the pull request's owner, repository, number, branch, head
|
||||
// and merge commit, which the gate's merges state holds. It reads the forge for every module directory (no kept inputs).
|
||||
func MergedMessage(owner, repo string, number int, branch, head, merge string, trees Trees, files Files) (Message, error) {
|
||||
body, err := json.Marshal(pull{Owner: owner, Repo: repo, Number: number, Head: branch, HeadSHA: head, Base: Trunk, MergeSHA: merge})
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
return ConvertKept(MergedSubject, body, trees, files, nil)
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the
|
||||
@@ -69,7 +69,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -90,7 +90,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil {
|
||||
if f, err = forgemsg.DecodeForge(msg.Data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(f.RetiredPlacements) > 0 || f.RetiredRead || strings.Contains(string(msg.Data), `"placements`) {
|
||||
@@ -108,7 +108,7 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
}
|
||||
forge.have[PredefinedFile] = []byte(`{"font-size":{"type":"integer","value":13}}`)
|
||||
msg, _ = Convert(mergedSubject, mergedBody(map[string]any{"repo": "mesh-catalog"}), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get)
|
||||
if f, err = delivery.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead {
|
||||
if f, err = forgemsg.DecodeForge(msg.Data); err != nil || string(f.Predefined) != string(forge.have[PredefinedFile]) || f.PredefinedRead {
|
||||
t.Errorf("predefined = %s (%v)", f.Predefined, err)
|
||||
}
|
||||
forge.calls = nil
|
||||
@@ -133,7 +133,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ := delivery.DecodeForge(msg.Data)
|
||||
f, _ := forgemsg.DecodeForge(msg.Data)
|
||||
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 1 || len(f.Packages) != 0 {
|
||||
t.Errorf("manifests %v locks %v packages %v", f.Manifests, f.Locks, f.Packages)
|
||||
}
|
||||
@@ -146,7 +146,7 @@ func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ = delivery.DecodeForge(msg.Data)
|
||||
f, _ = forgemsg.DecodeForge(msg.Data)
|
||||
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
|
||||
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
|
||||
}
|
||||
@@ -175,7 +175,7 @@ func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testin
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil {
|
||||
if f, _ := forgemsg.DecodeForge(msg.Data); f.Locks != nil {
|
||||
t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
)
|
||||
|
||||
// ADR 0495 clause 1 with the lead's decision L8 of R4e: the forge bridge is no reader of a manifest's `paths`. A merge of a
|
||||
@@ -26,7 +26,7 @@ func TestAMergedMessageCarriesNoNamedPathsTheBridgeDerived(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
f, err := forgemsg.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
// Package groups is what the forge bridge's process and its tools bundle share of a delivery group's end (novox/hq ADR
|
||||
// 0633 §4, issue 1585): the keys of the bridge's state performed, and the asks of the state end-asks, through which the
|
||||
// tool bridge_group_end_again asks the process to say a group's end again (the bridge serves no request subject, ADR 0437
|
||||
// §4, so a tool asks in a state the process watches, as indexer_take_in asks the indexer in takeins).
|
||||
package groups
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The bridge's states the tools read and write.
|
||||
const (
|
||||
// PerformedState is the bridge's state of what it performed and published.
|
||||
PerformedState = "performed"
|
||||
// EndAsksState is the state of asks to say a group's end again, keyed by the gate's merges key of the group.
|
||||
EndAsksState = "end-asks"
|
||||
// EndAsksBucket is its bucket on the bus.
|
||||
EndAsksBucket = delivery.ForgeBridgeModule + "_" + EndAsksState
|
||||
)
|
||||
|
||||
// Key is the key of performed that says a group's end is written: `groups.<merges key>`, its value the revision of the
|
||||
// gate's merges key it was worked out from.
|
||||
func Key(mergesKey string) string { return "groups." + mergesKey }
|
||||
|
||||
// PublishedKey is the key of performed that says a member's merged message is written: `published.<16 hex digits of the
|
||||
// sha256 of <repository>#<number>@<merge>>`.
|
||||
func PublishedKey(repository string, number int, merge string) string {
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge)))
|
||||
return "published." + hex.EncodeToString(sum[:8])
|
||||
}
|
||||
|
||||
// EndID is the message id of a group's end, and of one said again by hand at a time: the stream's duplicate window drops a
|
||||
// second publish of the same id.
|
||||
func EndID(group, lastMerge, again string) string {
|
||||
id := "group-end." + group + "." + lastMerge
|
||||
if again != "" {
|
||||
id += "." + again
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// AgainStamp is the time a hand ask appends to a group end's message id.
|
||||
func AgainStamp(at time.Time) string { return at.UTC().Format("20060102T150405.000000000Z") }
|
||||
|
||||
// EndAsk is a value of the end-asks state: which group's end a person asked to be said again, why, who and when, and, once
|
||||
// the bridge read it, its answer.
|
||||
type EndAsk struct {
|
||||
V int `json:"v"`
|
||||
Group string `json:"group"`
|
||||
Why string `json:"why,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Written bool `json:"written,omitempty"`
|
||||
Answer string `json:"answer,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
}
|
||||
|
||||
// Done says the ask is answered.
|
||||
func (a EndAsk) Done() bool { return !a.Answered.IsZero() }
|
||||
|
||||
// NewEndAsk is the value bridge_group_end_again writes for a group: its key (the gate's merges key) and the value.
|
||||
func NewEndAsk(group, why, by string, at time.Time) (string, []byte, error) {
|
||||
group = strings.TrimSpace(group)
|
||||
if group == "" {
|
||||
return "", nil, errors.New("name the group whose end is said again (group)")
|
||||
}
|
||||
key, err := delivery.MergesKey(group)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
data, err := json.Marshal(EndAsk{V: 1, Group: group, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return key, data, err
|
||||
}
|
||||
|
||||
// DecodeEndAsk reads an end-asks value.
|
||||
func DecodeEndAsk(data []byte) (EndAsk, error) {
|
||||
var a EndAsk
|
||||
if err := json.Unmarshal(data, &a); err != nil {
|
||||
return a, err
|
||||
}
|
||||
if a.V != 1 || a.Group == "" {
|
||||
return a, errors.New("an end-asks value of no version 1 naming no group")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Row is one group as bridge_groups answers it: the gate's merge of it and what the bridge kept of its end.
|
||||
type Row struct {
|
||||
Key string `json:"key"`
|
||||
Group string `json:"group"`
|
||||
Members []RowMember `json:"members"`
|
||||
Ended bool `json:"ended"`
|
||||
EndedAt uint64 `json:"ended-at-revision,omitempty"`
|
||||
Open bool `json:"open"`
|
||||
Words string `json:"words"`
|
||||
// LastEndAsk is the last ask to say the group's end again (bridge_group_end_again), its who, why and when, and the
|
||||
// bridge's answer once it read it; absent when none was asked. The state end-asks keeps the last per group.
|
||||
LastEndAsk *EndAsk `json:"last-end-ask,omitempty"`
|
||||
}
|
||||
|
||||
// RowMember is one approved member of a group: its merge's state at the gate and whether its merged message is written.
|
||||
type RowMember struct {
|
||||
Repository string `json:"repository"`
|
||||
Number int `json:"number"`
|
||||
State string `json:"state"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Published bool `json:"published"`
|
||||
}
|
||||
|
||||
// Rows are the groups the gate's merges hold with more than one member, each with what the bridge kept: ended (its end
|
||||
// written, the revision it was worked out from) or open (some member is not merged or its merged message not written, or its
|
||||
// end not written). merges is the gate's merges by key, performed the bridge's performed keys by key; unread are the gate's
|
||||
// keys that do not read. asks are the end-asks values by the same key; one that does not read is named unread as
|
||||
// `end-asks.<key>`.
|
||||
func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[string]uint64, asks map[string][]byte) (rows []Row, unread []string) {
|
||||
lastAsk := map[string]*EndAsk{}
|
||||
for key, raw := range asks {
|
||||
a, err := DecodeEndAsk(raw)
|
||||
if err != nil {
|
||||
unread = append(unread, EndAsksState+"."+key)
|
||||
continue
|
||||
}
|
||||
lastAsk[key] = &a
|
||||
}
|
||||
for key, raw := range merges {
|
||||
m, err := delivery.DecodeMerge(raw)
|
||||
if err != nil {
|
||||
unread = append(unread, key)
|
||||
continue
|
||||
}
|
||||
if len(m.Approved) < 2 {
|
||||
continue
|
||||
}
|
||||
r := Row{Key: key, Group: m.Group}
|
||||
allMerged := true
|
||||
for _, p := range m.Pulls {
|
||||
pub := p.Commit != "" && performed[PublishedKey(p.Repository, p.Number, p.Commit)]
|
||||
r.Members = append(r.Members, RowMember{Repository: p.Repository, Number: p.Number, State: string(p.State), Commit: p.Commit, Published: pub})
|
||||
if p.State != delivery.MergeMerged || !pub {
|
||||
allMerged = false
|
||||
}
|
||||
}
|
||||
if len(m.Pulls) < len(m.Approved) {
|
||||
allMerged = false
|
||||
}
|
||||
r.EndedAt, r.Ended = endedAt[key]
|
||||
r.LastEndAsk = lastAsk[key]
|
||||
r.Open = !r.Ended
|
||||
switch {
|
||||
case r.Ended:
|
||||
r.Words = "its end is written"
|
||||
case allMerged:
|
||||
r.Words = "every member is merged and its message written, and its end is not written: bridge_group_end_again says it"
|
||||
default:
|
||||
r.Words = "gathered: not every member is merged and its merged message written"
|
||||
}
|
||||
rows = append(rows, r)
|
||||
}
|
||||
sortRows(rows)
|
||||
sort.Strings(unread)
|
||||
return rows, unread
|
||||
}
|
||||
|
||||
func sortRows(rows []Row) { sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) }
|
||||
|
||||
// MembersOf are a merge's approved members with their heads, in the order of their keys.
|
||||
func MembersOf(m delivery.Merge) []delivery.Head {
|
||||
var out []delivery.Head
|
||||
for _, k := range slices.Sorted(func(yield func(string) bool) {
|
||||
for k := range m.Approved {
|
||||
if !yield(k) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}) {
|
||||
at := strings.LastIndex(k, "@")
|
||||
if at < 0 {
|
||||
continue
|
||||
}
|
||||
n, err := strconv.Atoi(k[at+1:])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, delivery.Head{Repository: k[:at], Number: n, Head: m.Approved[k]})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -28,12 +28,13 @@ type state struct {
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
Loads []string `json:"loads"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
@@ -44,6 +45,7 @@ type manifest struct {
|
||||
Streams []stream `json:"streams"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Tools []string `json:"tools"`
|
||||
Invokes []string `json:"invokes"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
@@ -107,11 +109,15 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
|
||||
// §5), the revision of each key of the gate's buckets the bridge performed, so a key is performed once per
|
||||
// revision (ADR 0437 §8; issue 1341), and what a merge read of each module directory by its tree, so a merge reads
|
||||
// the forge only for the directories it changed (issue 1126).
|
||||
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}}) { // 1279/AC6, 1341, 1126
|
||||
t.Errorf("state = %+v, want the buckets heads, performed and inputs", m.State)
|
||||
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}, {Name: "end-asks", History: 1}}) { // 1279/AC6, 1341, 1126, 1585
|
||||
t.Errorf("state = %+v, want the buckets heads, performed, inputs and end-asks", m.State)
|
||||
}
|
||||
// No other state, no bus, no seat, no tool, no event (ADR 0437 §4; ADR 0521: no check.status).
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
|
||||
// 1585/T18: the tools read the groups and ask the bridge to say a group's end again, through its state end-asks. The tool bridge_merged_message answers a merge's merged message again (issue 1585).
|
||||
if want := []string{"bridge_group_end_again", "bridge_groups", "bridge_merged_message"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||
}
|
||||
// No other state, no bus, no seat, no event (ADR 0437 §4; ADR 0521: no check.status).
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the bridge has none", k)
|
||||
}
|
||||
@@ -143,8 +149,8 @@ func TestManifestProcess(t *testing.T) { // 1235/AC1
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 1 {
|
||||
t.Fatalf("artifacts = %+v", m.Build.Artifacts)
|
||||
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "tools" || m.Build.Artifacts[1].From != "cmd/mesh-delivery-forge-bridge-tools" {
|
||||
t.Fatalf("artifacts = %+v, want the program and the tools bundle", m.Build.Artifacts)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-forge-bridge" || !strings.HasSuffix(a.From, a.Binary) {
|
||||
|
||||
@@ -13,7 +13,13 @@
|
||||
"state": [
|
||||
{ "name": "heads", "history": 1 },
|
||||
{ "name": "performed", "history": 1 },
|
||||
{ "name": "inputs", "history": 1 }
|
||||
{ "name": "inputs", "history": 1 },
|
||||
{ "name": "end-asks", "history": 1 }
|
||||
],
|
||||
"tools": [
|
||||
"bridge_groups",
|
||||
"bridge_group_end_again",
|
||||
"bridge_merged_message"
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.merges",
|
||||
@@ -75,6 +81,17 @@
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-forge-bridge",
|
||||
"binary": "mesh-delivery-forge-bridge"
|
||||
},
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-forge-bridge-tools",
|
||||
"binary": "mesh-delivery-forge-bridge-tools",
|
||||
"loads": [
|
||||
"mesh-delivery-forge-bridge-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -191,8 +192,33 @@ func heldCounts(s stdio.KeptState) func() (index.OldSectionsCount, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// keptRead is a state the tools bundle reads, as the index package reads a store: Get and Keys; it writes nothing.
|
||||
type keptRead struct{ s stdio.KeptState }
|
||||
|
||||
func (k keptRead) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := k.s.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value, e.Revision, true, nil
|
||||
}
|
||||
|
||||
func (k keptRead) Keys(context.Context) ([]string, error) { return k.s.Keys() }
|
||||
|
||||
func (keptRead) Create(context.Context, string, []byte) error { return errReadOnly }
|
||||
|
||||
func (keptRead) Update(context.Context, string, []byte, uint64) error { return errReadOnly }
|
||||
|
||||
func (keptRead) Delete(context.Context, string, uint64) error { return errReadOnly }
|
||||
|
||||
var errReadOnly = errors.New("the tools bundle only reads this state")
|
||||
|
||||
func main() {
|
||||
if err := stdio.Serve("", append(Tools(askTakeIn(stdio.State(index.TakeInsState), 25*time.Second), time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...)); err != nil {
|
||||
ask := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second)
|
||||
tools := append(Tools(ask, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...)
|
||||
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, ask, time.Now)...)
|
||||
tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, ask, time.Now)...)
|
||||
if err := stdio.Serve("", tools); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// mergedTool is the forge bridge's tool that answers a merge's merged message again (issue 1585): arguments repository
|
||||
// and merge, answer {message}, the message whole as the bridge writes it on FORGE. The indexer's manifest lists it in
|
||||
// invokes, which is the grant to call it.
|
||||
const mergedTool = delivery.ForgeBridgeModule + ".bridge_merged_message"
|
||||
|
||||
// messageVia asks the forge bridge for a merge's merged message, for a held merge the indexer does not hold in memory (a
|
||||
// restart, a new build). An error is the bridge's own words.
|
||||
func messageVia(call toolCaller) func(ctx context.Context, repository, merge string) ([]byte, error) {
|
||||
return func(_ context.Context, repository, merge string) ([]byte, error) {
|
||||
raw, err := call(mergedTool, map[string]string{"repository": repository, "merge": merge})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
Message json.RawMessage `json:"message"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || len(a.Message) == 0 {
|
||||
return nil, fmt.Errorf("the answer of %s is not {message}: %v", mergedTool, err)
|
||||
}
|
||||
return a.Message, nil
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,10 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -17,9 +20,15 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/graph"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
@@ -42,22 +51,89 @@ accounts {
|
||||
// bus with the stream FORGE and the indexer's durable <node>_<module> on it, the buckets the controller makes from the
|
||||
// manifest's state (mesh-delivery-indexer_candidates, history 10) and mesh-controller_builds holding build records, and
|
||||
// no mesh-delivery-gate_source bucket.
|
||||
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
|
||||
js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) {
|
||||
//
|
||||
// And the take-in step (ADR 0633, issue 1585 I8a, I8b): the indexer's states pending, verdicts, graph, taken and packages
|
||||
// and its stream take-in are there, mesh-build-go reported its toolchain, and the node-engine's composition answers the
|
||||
// build of the indexer that runs. A merge changing the indexer's own tree writes only the indexer's entry, its manifest,
|
||||
// its verdict and its key of the graph (and the packages it publishes); the rest is pending, said take-in.pending and
|
||||
// named by indexer_pending with the build it waits for. A group's first member is gathered meanwhile. Started again as
|
||||
// that build, the indexer takes the revision in and leaves the group gathered; the group's second member and its end
|
||||
// then make one take-in, said take-in.taken under the group's key.
|
||||
func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
|
||||
var running atomic.Pointer[string]
|
||||
first := "fp-the-indexer-that-runs"
|
||||
running.Store(&first)
|
||||
// The forge bridge's tool bridge_merged_message (issue 1585): answers the merged message published for a merge, by
|
||||
// repository@merge, as the bridge does; a restarted indexer holds none in memory and asks it.
|
||||
var bridgeMessages sync.Map
|
||||
js, ctx, candidates, _, restart := startIndexerRun(t, func(ctx context.Context, builds buildLog) {
|
||||
// The builder's log entries of two builds' ends (ADR 0571 clause 1), as the builder appends them.
|
||||
builds.end(t, "novox/mesh-delivery", "a", "modules/a")
|
||||
builds.end(t, "novox/mesh-catalog", "c", "modules/c")
|
||||
})
|
||||
h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef"
|
||||
for _, f := range []delivery.Forge{
|
||||
{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true},
|
||||
{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: delivery.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true},
|
||||
} {
|
||||
data, err := delivery.EncodeForge(f)
|
||||
}, func(ctx context.Context, js jetstream.JetStream) {
|
||||
// The states and the stream the controller makes from the indexer's manifest (old sections, ADR 0595 §11).
|
||||
for bucket, history := range map[string]uint8{index.PendingBucket: 1, delivery.VerdictsBucket: delivery.VerdictsHistory,
|
||||
graph.Bucket: 1, delivery.TakenBucket: 1, delivery.PackagesBucket: 1} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: history}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// The take-in events are the indexer's ordinary events (ADR 0633 §5), on the path manifest-accepted takes.
|
||||
for name, subject := range map[string]string{"INDEXER_EVENTS": "mesh.mod.mesh-delivery-indexer.event.>"} {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: name, Subjects: []string{subject}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event, _ := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
key, _ := delivery.ToolchainKey("n1", "go")
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tc.Put(ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The node-engine's composition on n1 (process.Conn.Call's subjects): the build of the indexer that runs there.
|
||||
answer := func(m *nats.Msg) {
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"runs": map[string]string{"fp": *running.Load()}}, "node": "n1"})
|
||||
_ = m.Respond(raw)
|
||||
}
|
||||
bridge := func(m *nats.Msg) {
|
||||
var ask struct{ Repository, Merge string }
|
||||
_ = json.Unmarshal(m.Data, &ask)
|
||||
data, ok := bridgeMessages.Load(ask.Repository + "@" + ask.Merge)
|
||||
if !ok {
|
||||
raw, _ := json.Marshal(map[string]any{"error": "the gate holds no merge of " + ask.Repository + " with that commit"})
|
||||
_ = m.Respond(raw)
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{"result": map[string]any{"message": json.RawMessage(data.([]byte))}, "node": "n1"})
|
||||
_ = m.Respond(raw)
|
||||
}
|
||||
for _, subject := range []string{"mesh.mod.mesh-delivery-forge-bridge.call.bridge_merged_message.>", "mesh.mod.mesh-delivery-forge-bridge.tool.bridge_merged_message"} {
|
||||
if _, err := js.Conn().Subscribe(subject, bridge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, subject := range []string{"mesh.seat.node-engine.call.composition.n1.>", "mesh.seat.node-engine.tool.composition.n1"} {
|
||||
if _, err := js.Conn().Subscribe(subject, answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
h1, h2 := "0123456789abcdef0123456789abcdef01234567", "89abcdef0123456789abcdef0123456789abcdef"
|
||||
for _, f := range []forgemsg.Forge{
|
||||
{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1, Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true},
|
||||
{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 300, Head: h2, Action: forgemsg.PullUpdated, Dirs: []string{"modules/c"}, DirsKnown: true},
|
||||
} {
|
||||
data, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgePull)
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -83,11 +159,202 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
|
||||
t.Errorf("%s#%d modules = %v, want [%s]", p.Repository, p.Number, p.Modules, want)
|
||||
}
|
||||
}
|
||||
|
||||
// I8a: pull request 12 merged, changing the indexer's own tree and a's.
|
||||
manifest, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const ix = delivery.IndexerModule
|
||||
m1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h1,
|
||||
Merge: "1111111111111111111111111111111111111111", Root: "r0",
|
||||
Trees: map[string]string{"modules/" + ix: "tree-indexer-2", "modules/a": "tree-a-2"},
|
||||
Manifests: map[string]string{"modules/" + ix: string(manifest), "modules/a": `{"module":"a","version":"1"}`},
|
||||
Locks: map[string]map[string]string{"modules/" + ix: {"go.sum": "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"}}}
|
||||
merged := func(f forgemsg.Forge) uint64 {
|
||||
t.Helper()
|
||||
data, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
event, _ := forgemsg.ForgeEvent(f.Repository, forgemsg.ForgeMerged)
|
||||
bridgeMessages.Store(f.Repository+"@"+f.Merge, data)
|
||||
ack, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+event, data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ack.Sequence
|
||||
}
|
||||
m1Key := takein.PendingKey(merged(m1))
|
||||
events, err := js.Stream(ctx, "INDEXER_EVENTS")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// saidAll is every take-in event of kind said under key so far, oldest first.
|
||||
saidAll := func(kind, key string) []takein.TakeIn {
|
||||
event, _ := takein.TakeInEvent(kind)
|
||||
info, err := events.Info(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []takein.TakeIn
|
||||
for seq := info.State.FirstSeq; seq <= info.State.LastSeq && seq > 0; seq++ {
|
||||
msg, err := events.GetMsg(ctx, seq)
|
||||
if err != nil || msg.Subject != "mesh.mod."+ix+".event."+event {
|
||||
continue
|
||||
}
|
||||
if ti, err := takein.DecodeTakeIn(msg.Data); err == nil && ti.Key == key {
|
||||
out = append(out, ti)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
// said is the last take-in event under kind and key, within 20 s.
|
||||
said := func(kind, key string) (takein.TakeIn, bool) {
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if all := saidAll(kind, key); len(all) > 0 {
|
||||
return all[len(all)-1], true
|
||||
}
|
||||
}
|
||||
return takein.TakeIn{}, false
|
||||
}
|
||||
held, ok := said(takein.TakeInPending, m1Key)
|
||||
if !ok || held.What != takein.TakeInRevision || held.WaitsFor == "" || held.WaitsFor == first || len(held.Merges) != 1 {
|
||||
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the revision waiting for another build than %s", m1Key, held, ok, first)
|
||||
}
|
||||
named := held.WaitsFor
|
||||
get := func(bucket, key string) ([]byte, bool) {
|
||||
t.Helper()
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return e.Value(), true
|
||||
}
|
||||
// Only what building and moving the named indexer needs is written.
|
||||
raw, ok := get(delivery.ModulesBucket, ix)
|
||||
own, err := delivery.DecodeModule(raw)
|
||||
if !ok || err != nil || own.FP2 != named || own.Commit != m1.Merge {
|
||||
t.Fatalf("modules.%s = %+v (%v), want the named build %s at the merge", ix, own, err, named)
|
||||
}
|
||||
if _, ok := get(delivery.ManifestsBucket, own.Manifest); !ok {
|
||||
t.Errorf("manifests.%s is not written", own.Manifest)
|
||||
}
|
||||
if _, ok := get(delivery.VerdictsBucket, own.Manifest); !ok {
|
||||
t.Errorf("the verdict on the indexer's manifest %s is not written", own.Manifest)
|
||||
}
|
||||
if gk, _ := graph.Key(ix); !func() bool { _, ok := get(graph.Bucket, gk); return ok }() {
|
||||
t.Errorf("the indexer's key %s of the module graph is not written", gk)
|
||||
}
|
||||
repoKey, _ := delivery.RepositoryKey(m1.Repository)
|
||||
if _, ok := get(delivery.ModulesBucket, "a"); ok {
|
||||
t.Errorf("modules.a is written while the revision is pending")
|
||||
}
|
||||
if _, ok := get(delivery.TakenBucket, repoKey); ok {
|
||||
t.Errorf("taken.%s is written while the revision is pending", repoKey)
|
||||
}
|
||||
if e, err := candidates.Get(ctx, "feat/x"); err != nil {
|
||||
t.Errorf("the candidate feat/x is gone while the merge of #12 is pending: %v", err)
|
||||
} else if c, err := delivery.DecodeCandidate(e.Value()); err != nil || len(c.Pulls) != 2 {
|
||||
t.Errorf("the candidate feat/x = %+v while the merge of #12 is pending, want its two pull requests", c)
|
||||
}
|
||||
|
||||
// A group's first member arrives meanwhile; its end is still to come (the knowledge base's remark: it stays across the restart).
|
||||
members := []delivery.Head{{Repository: "novox/mesh-catalog", Number: 40, Head: "4040404040404040404040404040404040404040"},
|
||||
{Repository: "novox/mesh-controller", Number: 41, Head: "4141414141414141414141414141414141414141"}}
|
||||
g1 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/g", Number: 40, Head: members[0].Head,
|
||||
Merge: "4000000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/c": "tree-c-2"},
|
||||
Manifests: map[string]string{"modules/c": `{"module":"c","version":"1"}`}, Group: "feat/g", Members: members}
|
||||
g2 := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-controller", Branch: "feat/g", Number: 41, Head: members[1].Head,
|
||||
Merge: "4100000000000000000000000000000000000000", Root: "r0", Trees: map[string]string{"modules/k": "tree-k-1"},
|
||||
Manifests: map[string]string{"modules/k": `{"module":"k","version":"1"}`}, Group: "feat/g", Members: members}
|
||||
g1Key, groupKey := takein.PendingKey(merged(g1)), takein.GroupTakeInKey("feat/g")
|
||||
if gathered, ok := said(takein.TakeInPending, groupKey); !ok || gathered.What != takein.TakeInGroup || len(gathered.Merges) != 1 {
|
||||
t.Fatalf("take-in.pending.%s = %+v (said %v) within 20 s, want the group with its first member", groupKey, gathered, ok)
|
||||
}
|
||||
// indexer_pending, the tool over the same state, names both waits.
|
||||
pendingKV, err := js.KeyValue(ctx, index.PendingBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waits := func() []takein.PendingWait {
|
||||
t.Helper()
|
||||
answer, err := index.PendingTools(kvStore{pendingKV}, nil, nil, time.Now)[0].Run(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return answer.(takein.PendingAnswer).Waits
|
||||
}
|
||||
var w []takein.PendingWait
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if w = waits(); len(w) == 2 && w[0].WaitsFor == named && w[1].Said != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(w) != 2 || w[0].Key != m1Key || w[0].WaitsFor != named || !slices.Contains(w[0].Holds, "a") || !slices.Contains(w[0].Holds, ix) ||
|
||||
w[1].Key != groupKey || w[1].What != takein.TakeInGroup || len(w[1].Merges) != 1 || w[1].Said != takein.TakeInPendingEvent {
|
||||
t.Fatalf("indexer_pending = %+v, want the revision %s waiting for %s holding a and the indexer, then the group gathered", w, m1Key, named)
|
||||
}
|
||||
|
||||
// The named indexer starts (the node-engine now runs that build): it takes the revision in, and leaves the group.
|
||||
running.Store(&named)
|
||||
restart()
|
||||
if taken, ok := said(takein.TakeInTaken, m1Key); !ok || len(taken.Merges) != 1 || taken.Merges[0].Merge != m1.Merge {
|
||||
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the start of %s", m1Key, taken, ok, named)
|
||||
}
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if w = waits(); len(w) == 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw, ok := get(delivery.ModulesBucket, "a"); !ok || !strings.Contains(string(raw), m1.Merge) {
|
||||
t.Errorf("modules.a = %s after the take-in at the start, want it at the merge %s", raw, m1.Merge)
|
||||
}
|
||||
if _, ok := get(delivery.TakenBucket, repoKey); !ok {
|
||||
t.Errorf("taken.%s is not written by the take-in at the start", repoKey)
|
||||
}
|
||||
if len(w) != 1 || w[0].Key != groupKey || len(w[0].Keys) != 1 || w[0].Keys[0] != g1Key {
|
||||
t.Fatalf("indexer_pending after the start = %+v, want only the group gathered, its first member %s", w, g1Key)
|
||||
}
|
||||
if _, ok := get(delivery.ModulesBucket, "c"); ok {
|
||||
t.Errorf("modules.c is written while its group is gathered")
|
||||
}
|
||||
|
||||
// I8b: the group's second member and its end: one take-in of both, said under the group's key.
|
||||
merged(g2)
|
||||
end, err := forgemsg.EncodeForgeGroupEnd(forgemsg.ForgeGroupEnd{Group: "feat/g", Members: []forgemsg.GroupMerge{
|
||||
{Repository: g1.Repository, Number: g1.Number, Merge: g1.Merge}, {Repository: g2.Repository, Number: g2.Number, Merge: g2.Merge}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event."+forgemsg.ForgeGroupEndEvent, end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if taken, ok := said(takein.TakeInTaken, groupKey); !ok || taken.What != takein.TakeInGroup || len(taken.Merges) != 2 {
|
||||
t.Fatalf("take-in.taken.%s = %+v (said %v) within 20 s of the group's end, want both members", groupKey, taken, ok)
|
||||
}
|
||||
var hasC, hasK bool
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !(hasC && hasK && len(w) == 0); time.Sleep(50 * time.Millisecond) {
|
||||
_, hasC = get(delivery.ModulesBucket, "c")
|
||||
_, hasK = get(delivery.ModulesBucket, "k")
|
||||
w = waits()
|
||||
}
|
||||
if !hasC || !hasK || len(w) != 0 {
|
||||
t.Errorf("after the group's end: modules.c %v, modules.k %v, indexer_pending %+v: want both taken in and nothing waiting", hasC, hasK, w)
|
||||
}
|
||||
// One take-in: neither member was said as a revision of its own.
|
||||
if len(saidAll(takein.TakeInPending, g1Key)) != 0 {
|
||||
t.Errorf("the group's first member %s was said pending as a revision of its own", g1Key)
|
||||
}
|
||||
}
|
||||
|
||||
// startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
|
||||
// mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created.
|
||||
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
|
||||
// startIndexerRun starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
|
||||
// mesh-controller_builds, and runs the program; restart starts it again. mesh-delivery-gate_source is never created.
|
||||
func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue, func()) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -166,11 +433,24 @@ func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...f
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
var stop func()
|
||||
start := func() {
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
stop = func() { cancel(); <-done }
|
||||
}
|
||||
start()
|
||||
t.Cleanup(func() { stop() })
|
||||
// restart stops the program and starts it again on the same bus, as the node-engine does with a new build.
|
||||
restart := func() { stop(); start() }
|
||||
|
||||
return js, ctx, candidates, modules, restart
|
||||
}
|
||||
|
||||
// startIndexer is startIndexerRun for a test that never starts the program again.
|
||||
func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
|
||||
js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...)
|
||||
return js, ctx, candidates, modules
|
||||
}
|
||||
|
||||
@@ -202,7 +482,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the m
|
||||
builds.end(t, "novox/mesh-controller", "build-agent", "modules/build-agent")
|
||||
})
|
||||
const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: merge, Root: "r0", Trees: map[string]string{"modules/a": "newtree"},
|
||||
Manifests: map[string]string{"modules/a": `{"module":"a","version":"1"}`}})
|
||||
if err != nil {
|
||||
@@ -256,7 +536,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
forge := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
forge := forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
@@ -264,7 +544,7 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4, 15
|
||||
for name, raw := range judged {
|
||||
forge.Manifests["modules/"+name], forge.Trees["modules/"+name] = raw, "tree-"+name
|
||||
}
|
||||
data, err := delivery.EncodeForge(forge)
|
||||
data, err := forgemsg.EncodeForge(forge)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -358,7 +638,7 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes
|
||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds buildLog) {
|
||||
builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi")
|
||||
})
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
@@ -445,7 +725,7 @@ func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBuc
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
pull, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: delivery.PullUpdated,
|
||||
pull, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: forgemsg.PullUpdated,
|
||||
Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": string(manifest)}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -615,7 +895,7 @@ func TestIndexerCountsOnlyThePlacedBuildNodesToolchains(t *testing.T) {
|
||||
place(ctx, t, "n1", "mesh-build-go")
|
||||
place(ctx, t, "n2")
|
||||
})
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
|
||||
data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
@@ -153,6 +154,34 @@ type delivered interface {
|
||||
Metadata() (*jetstream.MsgMetadata, error)
|
||||
}
|
||||
|
||||
// groupEndSubject is the whole subject of a delivery group's end on the forge bridge's stream: compared exactly, so a
|
||||
// repository named forge/group, whose merged message ends the same way, is never taken for one.
|
||||
var groupEndSubject = process.ModuleStreamSubject(delivery.ForgeBridgeModule, forgemsg.ForgeGroupEndEvent)
|
||||
|
||||
// handleGroupEnd takes a group's end in (index.EndGroup): a body that does not read is terminated and said; an end
|
||||
// whose members are not all held yet, or while the build this program runs is not known, is left for redelivery.
|
||||
func handleGroupEnd(ctx context.Context, m delivered, ix *index.Indexer, h *index.Heard, logger *log.Logger) {
|
||||
end, err := forgemsg.DecodeForgeGroupEnd(m.Data())
|
||||
if err != nil {
|
||||
logger.Printf("%s: not a group's end, dropped: %v", m.Subject(), err)
|
||||
h.Outcome, h.Words = index.HeardDropped, err.Error()
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
h.Kind, h.Branch = forgemsg.ForgeGroupEndEvent, end.Group
|
||||
applied, words, err := ix.EndGroup(ctx, end)
|
||||
h.Written, h.Late = applied.Written, applied.Late
|
||||
if err != nil {
|
||||
logger.Printf("%s: the end of the group %q left for redelivery: %v", m.Subject(), end.Group, err)
|
||||
h.Outcome, h.Words = index.HeardLeft, err.Error()
|
||||
return
|
||||
}
|
||||
h.Outcome, h.Words = index.HeardApplied, words
|
||||
if err := m.DoubleAck(ctx); err != nil {
|
||||
logger.Printf("%s: the end of the group %q applied, its acknowledgement was not confirmed: %v", m.Subject(), end.Group, err)
|
||||
}
|
||||
}
|
||||
|
||||
// handle takes one message of FORGE into the candidates and acknowledges it. A message it can never read is terminated,
|
||||
// never redelivered, and the log names its subject and why. A write that failed stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock). Each message handled is written to the heard state
|
||||
@@ -171,7 +200,12 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto
|
||||
logger.Printf("%s: sequence %d is not written to the heard state: %v", h.Subject, h.Sequence, err)
|
||||
}
|
||||
}()
|
||||
f, err := delivery.DecodeForge(m.Data())
|
||||
// A delivery group's end (novox/hq ADR 0633 §4) has a body of its own: told apart by its name, before the decode.
|
||||
if m.Subject() == groupEndSubject {
|
||||
handleGroupEnd(ctx, m, ix, &h, logger)
|
||||
return
|
||||
}
|
||||
f, err := forgemsg.DecodeForge(m.Data())
|
||||
if err != nil {
|
||||
logger.Printf("%s: not a forge message, dropped: %v", m.Subject(), err)
|
||||
h.Outcome, h.Words = index.HeardDropped, err.Error()
|
||||
@@ -182,7 +216,13 @@ func handle(ctx context.Context, m delivered, ix *index.Indexer, heard index.Sto
|
||||
if f.Merge != "" {
|
||||
h.Commit = f.Merge
|
||||
}
|
||||
applied, err := ix.ApplyMessage(ctx, f)
|
||||
var applied index.Applied
|
||||
if f.Kind == forgemsg.ForgeMerged && h.Sequence > 0 {
|
||||
// A merge is taken in only by the indexer its revision names (ADR 0633 §1): kept in pending until then.
|
||||
applied, h.Words, err = ix.Arrive(ctx, h.Sequence, f)
|
||||
} else {
|
||||
applied, err = ix.ApplyMessage(ctx, f)
|
||||
}
|
||||
h.Written, h.Late = applied.Written, applied.Late
|
||||
switch {
|
||||
case errors.Is(err, index.ErrUnreadable):
|
||||
@@ -303,12 +343,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
|
||||
// The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written.
|
||||
graph := process.AwaitBucket(ctx, js, conn, graph.Bucket, "no module graph is written", logger.Printf)
|
||||
ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
|
||||
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
|
||||
tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf)
|
||||
ix := &index.Indexer{OwnBuildState: awaitedStore{ownBuildState}, TryRecovery: awaitedStore{tryRecovery}, OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
|
||||
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
|
||||
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
|
||||
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}}
|
||||
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), ReadFile: readFileVia(calls.Call),
|
||||
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), Graph: awaitedStore{graph}, Message: messageVia(calls.Call),
|
||||
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
|
||||
// until then every merge is taken in at once, as before.
|
||||
Pending: awaitedStore{process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
|
||||
SayTakeIn: sayTakeIn(js, cred)}
|
||||
// Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the
|
||||
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
|
||||
// revisions in with its old judge.
|
||||
ix.FixOwnBuild(ctx)
|
||||
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
|
||||
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
|
||||
var started atomic.Bool
|
||||
@@ -368,7 +418,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if !gate.Toolchain(module, key, value, deleted) {
|
||||
return nil
|
||||
}
|
||||
return ix.Recompute(ctx)
|
||||
// The modules written again, and pending evaluated again (ADR 0633 §1): the build a revision names moves with
|
||||
// the toolchain.
|
||||
return ix.ToolchainReported(ctx)
|
||||
},
|
||||
recompute: ix.Recompute}, logger)
|
||||
if err := tools.start(); err != nil {
|
||||
@@ -448,11 +500,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
defer tc.Stop()
|
||||
}
|
||||
switch pcons, err := cred.EventConsumer(ctx, js, PreviewsStream); {
|
||||
case err != nil:
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err)
|
||||
default:
|
||||
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.Written(m) }, logger.Printf)
|
||||
// The consumer on the checker's stream exists before the open tries are listed and is heard after (ADR 0638 §3): an event
|
||||
// said between the call and the hearing waits in it, and a try that started or ended meanwhile is found either way.
|
||||
pcons, perr := cred.EventConsumer(ctx, js, PreviewsStream)
|
||||
if perr != nil {
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews and tries are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, perr)
|
||||
}
|
||||
// The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's
|
||||
// stream starts (ADR 0633 §2). When the build this program runs is not known yet, nothing waits for it: the consumer
|
||||
// starts, and a merged message left unacknowledged asks the build again at its redelivery (clause 1). A hand take-in
|
||||
// (indexer_take_in_pending) is answered at any time: the takeins watch runs already.
|
||||
if _, words, err := ix.TakeInAtStart(ctx); err != nil {
|
||||
return fmt.Errorf("the take-in at start: %w", err)
|
||||
} else if words != "" {
|
||||
logger.Printf("take-in at start: %s", words)
|
||||
}
|
||||
// The open tries are found again through the checker's tool: a try event missed while the indexer was down would leave a
|
||||
// try with no candidate, or a candidate with no try. Not fatal: the events come, and a preview written writes the candidates.
|
||||
// A recovery that fails is said (journal, and indexer_try_candidates) and the events are heard all the same.
|
||||
_ = ix.RecoverTries(ctx)
|
||||
if perr == nil {
|
||||
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -73,8 +74,8 @@ const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
st := &memStore{vals: map[string][]byte{}}
|
||||
ix := &index.Indexer{Store: st, Builds: index.NewBuilds()}
|
||||
data, _ := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
|
||||
data, _ := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: forgemsg.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true})
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull", data: data}
|
||||
heard := &memStore{vals: map[string][]byte{}}
|
||||
handle(context.Background(), m, ix, heard, log.New(&bytes.Buffer{}, "", 0))
|
||||
@@ -84,7 +85,7 @@ func TestAMessageIsIndexedAndAcked(t *testing.T) { // 1238/AC2 (unit part)
|
||||
}
|
||||
// Issue 1126 note 76: what the indexer heard of FORGE is readable (indexer_heard): subject, sequence, deliveries, time,
|
||||
// repository, commit and what it did.
|
||||
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != delivery.ForgePull ||
|
||||
if h := heardOf(t, heard); h.Subject != m.subject || h.Sequence != 41 || h.Delivered != 2 || h.Published.Hour() != 13 || h.Kind != forgemsg.ForgePull ||
|
||||
h.Repository != "novox/mesh-delivery" || h.Number != 12 || h.Commit != head || h.Outcome != index.HeardApplied || h.Handled.IsZero() {
|
||||
t.Errorf("heard = %+v", h)
|
||||
}
|
||||
@@ -102,6 +103,31 @@ func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 12
|
||||
if h := heardOf(t, heard); h.Outcome != index.HeardDropped || h.Sequence != 41 || h.Words == "" {
|
||||
t.Errorf("heard = %+v", h)
|
||||
}
|
||||
// 1585/T9 (ADR 0633 §4): a group's end is told apart by its name before the forge decode: one that reads is acked
|
||||
// (an indexer with no pending state takes the members in one by one), one that does not is dropped as a group's end.
|
||||
for _, c := range []struct {
|
||||
data string
|
||||
termd int
|
||||
logged string
|
||||
}{{`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`, 0, ""},
|
||||
{`{"v":1,"group":"feat/x"}`, 1, "not a group's end"}} {
|
||||
logs.Reset()
|
||||
m := &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event." + forgemsg.ForgeGroupEndEvent, data: []byte(c.data)}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
|
||||
if m.termd != c.termd || m.acked+m.termd != 1 || !strings.Contains(logs.String(), c.logged) {
|
||||
t.Errorf("group end %s: terminated %d acked %d log %q", c.data, m.termd, m.acked, logs.String())
|
||||
}
|
||||
if h := heardOf(t, heard); h.Kind != forgemsg.ForgeGroupEndEvent && c.termd == 0 {
|
||||
t.Errorf("heard of a group's end = %+v", h)
|
||||
}
|
||||
}
|
||||
// The repository forge/group's merged message ends as a group's end does: it is a forge message all the same.
|
||||
logs.Reset()
|
||||
m = &fakeMsg{subject: "mesh.stream.mesh-delivery-forge-bridge.event.forge.forge.group.merged", data: []byte(`{"v":1,"group":"feat/x","members":[{"repository":"novox/mesh-sdk","number":3,"merge":"` + head + `"}]}`)}
|
||||
handle(context.Background(), m, &index.Indexer{Store: st, Builds: index.NewBuilds()}, heard, log.New(&logs, "", 0))
|
||||
if !strings.Contains(logs.String(), "not a forge message") {
|
||||
t.Errorf("forge/group's merged message read as a group's end: %q", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value)
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
@@ -18,7 +20,7 @@ import (
|
||||
// commitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea): with `paths`, by module directory
|
||||
// (the root spelled ""), the paths relative to it, it answers {root, trees, paths} with paths by directory then path, each
|
||||
// path's git object id; a path not at the commit fails the call naming it (the interface agreed in R4e; mesh-sdk
|
||||
// delivery.Forge.Paths; novox/hq ADR 0499 clause 3).
|
||||
// forgemsg.Forge.Paths; novox/hq ADR 0499 clause 3).
|
||||
const commitTreesTool = "gitea.gitea_commit_trees"
|
||||
|
||||
// toolCaller calls a module's tool, `<module>.<tool>`, as the indexer's own account (mesh-sdk process.Conn.Call).
|
||||
@@ -100,7 +102,7 @@ func filesVia(call toolCaller) func(ctx context.Context, repository, sha string,
|
||||
}
|
||||
for _, d := range dirs {
|
||||
if _, ok := a.Files[d]; !ok {
|
||||
return nil, fmt.Errorf("the answer of %s lists no files of %q", commitTreesTool, d)
|
||||
return nil, fmt.Errorf("the answer of %s lists no files of %q: %w", commitTreesTool, d, index.ErrNotInTree)
|
||||
}
|
||||
}
|
||||
return a.Files, nil
|
||||
@@ -119,6 +121,12 @@ func goModVia(call toolCaller) func(ctx context.Context, repository, sha, dir st
|
||||
return goFileVia(call, "go.mod")
|
||||
}
|
||||
|
||||
// readFileVia reads one file of a repository at a commit by its path (gitea_get_file); a file the commit does not hold is
|
||||
// none. The seat definitions of a revision are read with it (seatsof.go).
|
||||
func readFileVia(call toolCaller) func(ctx context.Context, repository, sha, path string) ([]byte, error) {
|
||||
return goFileVia(call, "")
|
||||
}
|
||||
|
||||
// goFileVia reads one Go file of a module directory at a commit through the forge's get-file tool; a file the commit does
|
||||
// not hold is none.
|
||||
func goFileVia(call toolCaller, name string) func(ctx context.Context, repository, sha, dir string) ([]byte, error) {
|
||||
@@ -128,7 +136,10 @@ func goFileVia(call toolCaller, name string) func(ctx context.Context, repositor
|
||||
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||
}
|
||||
p := name
|
||||
if dir != "" && dir != delivery.RootPath {
|
||||
switch {
|
||||
case name == "": // readFileVia: dir is the file's whole path
|
||||
p = dir
|
||||
case dir != "" && dir != delivery.RootPath:
|
||||
p = dir + "/" + name
|
||||
}
|
||||
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
|
||||
@@ -258,8 +269,8 @@ const (
|
||||
)
|
||||
|
||||
// openVia asks the forge which pull requests into the trunk are open now (gitea_open_pulls), every repository's.
|
||||
func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, error) {
|
||||
return func(context.Context) ([]delivery.OpenPull, error) {
|
||||
func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, error) {
|
||||
return func(context.Context) ([]forgemsg.OpenPull, error) {
|
||||
raw, err := call(openPullsTool, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -275,12 +286,40 @@ func openVia(call toolCaller) func(ctx context.Context) ([]delivery.OpenPull, er
|
||||
if err := json.Unmarshal(raw, &o); err != nil || o.Pulls == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {pulls}: %v", openPullsTool, err)
|
||||
}
|
||||
out := []delivery.OpenPull{}
|
||||
out := []forgemsg.OpenPull{}
|
||||
for _, p := range *o.Pulls {
|
||||
if p.Base == trunkBranch && p.Number > 0 {
|
||||
out = append(out, delivery.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
|
||||
out = append(out, forgemsg.OpenPull{Repository: p.Owner + "/" + p.Repo, Number: p.Number})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// openTriesTool is the checker's tool that lists the tries that wait or run (mesh-delivery-checker checker_tries).
|
||||
const openTriesTool = "mesh-delivery-checker.checker_tries"
|
||||
|
||||
// openTriesVia lists the open tries through the checker's tool, never its state (ADR 0596).
|
||||
func openTriesVia(call toolCaller) func(ctx context.Context) ([]tryevent.Open, error) {
|
||||
return func(context.Context) ([]tryevent.Open, error) {
|
||||
raw, err := call(openTriesTool, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// No try open is an answer: the checker's list is empty, or null in an older checker (issue 1921). An answer without the
|
||||
// key is none.
|
||||
var o map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &o); err != nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err)
|
||||
}
|
||||
list, ok := o["tries"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the answer of %s is not {tries}: no tries key", openTriesTool)
|
||||
}
|
||||
var tries []tryevent.Open
|
||||
if err := json.Unmarshal(list, &tries); err != nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err)
|
||||
}
|
||||
return tries, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -50,6 +51,15 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
if _, has := gotArgs.(map[string]any)["paths"]; has {
|
||||
t.Error("the trunk asked for paths")
|
||||
}
|
||||
// A directory the answer lists no files of is a failure of the tree (index.ErrNotInTree); a call that fails is not.
|
||||
noDir := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"files":{}}`), nil }
|
||||
if _, err := filesVia(noDir)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); !errors.Is(err, index.ErrNotInTree) {
|
||||
t.Errorf("a missing directory: %v, want ErrNotInTree", err)
|
||||
}
|
||||
down := func(string, any) (json.RawMessage, error) { return nil, errors.New("down") }
|
||||
if _, err := filesVia(down)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); err == nil || errors.Is(err, index.ErrNotInTree) {
|
||||
t.Errorf("a failed call: %v, want an error that is not ErrNotInTree", err)
|
||||
}
|
||||
// The build the indexer runs is the engine's word (Runs.FP of its composition verb), asked of its own machine.
|
||||
var asked string
|
||||
own := func(key string, body any) (json.RawMessage, error) {
|
||||
@@ -59,10 +69,45 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3
|
||||
if fp, err := ownBuildVia(own, "n1")(context.Background()); err != nil || fp != "src2:abc" || asked != "seat:node-engine.composition@n1" {
|
||||
t.Errorf("own build = %q %v asked %q", fp, err, asked)
|
||||
}
|
||||
// Issue 1921: the start during a replacement. What runs still names the old build, the want (composed by the node itself)
|
||||
// the successor, which this process is: it fixes the successor. A want that is not self, or is removed, leaves what runs.
|
||||
for _, row := range []struct{ name, answer, want string }{
|
||||
{"replacing: runs old, want new and self", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true}}`, "src2:new"},
|
||||
{"a want the node does not compose", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new"}}`, "src2:old"},
|
||||
{"replacing: the apply of the want failed or is held", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:new","held":true}}`, "src2:old"},
|
||||
{"replacing: an attempt of another build", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:older"}}`, "src2:new"},
|
||||
{"the want already runs", `{"runs":{"fp":"src2:new"},"want":{"fp":"src2:new","self":true}}`, "src2:new"},
|
||||
{"a removed want", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true,"removed":true}}`, "src2:old"},
|
||||
{"no build runs yet, want self", `{"want":{"fp":"src2:new","self":true}}`, "src2:new"},
|
||||
} {
|
||||
ask := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil }
|
||||
if fp, err := ownBuildVia(ask, "n1")(context.Background()); err != nil || fp != row.want {
|
||||
t.Errorf("%s: own build = %q %v, want %q", row.name, fp, err, row.want)
|
||||
}
|
||||
}
|
||||
none := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }
|
||||
if _, err := ownBuildVia(none, "n1")(context.Background()); err == nil {
|
||||
t.Error("an engine that says nothing runs gave a build")
|
||||
}
|
||||
// Issue 1921: the checker's list of open tries; none open is an answer, whether [] or null.
|
||||
for _, row := range []struct {
|
||||
name, answer string
|
||||
want int
|
||||
refused bool
|
||||
}{
|
||||
{"one open", `{"tries":[{"id":"try-1","state":"running","whole":true,"members":[{"repository":"novox/a","head":"h","trunk":"t"}]}]}`, 1, false},
|
||||
{"an empty list", `{"tries":[]}`, 0, false},
|
||||
{"null, as an older checker answers none", `{"tries":null}`, 0, false},
|
||||
{"no tries key", `{}`, 0, true},
|
||||
{"not json", `nope`, 0, true},
|
||||
{"tries of another shape", `{"tries":{"a":1}}`, 0, true},
|
||||
} {
|
||||
call := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil }
|
||||
got, err := openTriesVia(call)(context.Background())
|
||||
if (err != nil) != row.refused || len(got) != row.want {
|
||||
t.Errorf("%s: %d tries, error %v; want %d, refused %v", row.name, len(got), err, row.want, row.refused)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadViaReadsManifestsAndLocks(t *testing.T) {
|
||||
|
||||
@@ -25,10 +25,32 @@ func ownBuildVia(call toolCaller, node string) func(ctx context.Context) (string
|
||||
Runs *struct {
|
||||
FP string `json:"fp"`
|
||||
} `json:"runs"`
|
||||
Want *struct {
|
||||
FP string `json:"fp"`
|
||||
Self bool `json:"self"`
|
||||
Removed bool `json:"removed"`
|
||||
} `json:"want"`
|
||||
// Attempt is the engine's last apply that did not end running (failed, or held), with the build it was of.
|
||||
Attempt *struct {
|
||||
FP string `json:"fp"`
|
||||
} `json:"attempt"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return "", fmt.Errorf("the answer of %s is not a composition: %w", compositionVerb, err)
|
||||
}
|
||||
// While the engine's apply of the want is in progress (what runs still names the old build, the want names another, and no
|
||||
// attempt names that want as failed or held), this process is the successor and fixes the want's build. self is the node's
|
||||
// standing way of composing the module, not a mark of this process, so it is not enough alone: an old process started again
|
||||
// while the want names a successor that does not run (its apply failed or is held) takes what runs. The start fixes the
|
||||
// build once (quick fix, ADR 0639 §7: nothing is learnt later, nothing waits on a timer); the thorough form is the
|
||||
// engine passing each process the fingerprint it starts it with.
|
||||
running := ""
|
||||
if a.Runs != nil {
|
||||
running = a.Runs.FP
|
||||
}
|
||||
if w := a.Want; w != nil && w.Self && !w.Removed && w.FP != "" && w.FP != running && (a.Attempt == nil || a.Attempt.FP != w.FP) {
|
||||
return w.FP, nil
|
||||
}
|
||||
if a.Runs == nil || a.Runs.FP == "" {
|
||||
return "", errors.New("the engine says no fingerprint of the indexer runs")
|
||||
}
|
||||
|
||||
+8
-7
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -71,10 +72,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
|
||||
}
|
||||
serveForge(t, js.Conn(), listing, goSumContent)
|
||||
})
|
||||
merged := func(repo string, number int, merge string, f delivery.Forge) {
|
||||
merged := func(repo string, number int, merge string, f forgemsg.Forge) {
|
||||
t.Helper()
|
||||
f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = delivery.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge
|
||||
data, err := delivery.EncodeForge(f)
|
||||
f.Kind, f.Repository, f.Branch, f.Number, f.Head, f.Merge = forgemsg.ForgeMerged, repo, "feat/x", number, "0123456789abcdef0123456789abcdef01234567", merge
|
||||
data, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -101,10 +102,10 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
|
||||
"tsconfig.json": "5555555555555555555555555555555555555555"}}
|
||||
}
|
||||
// mesh-sdk at m1, then the catalogue's two modules: their go.sum names mesh-sdk, read by the indexer itself.
|
||||
merged("novox/mesh-sdk", 70, m1, delivery.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1},
|
||||
merged("novox/mesh-sdk", 70, m1, forgemsg.Forge{Root: sdkRoot1, Trees: map[string]string{"": sdkRoot1},
|
||||
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("6666666666666666666666666666666666666666")})
|
||||
module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m1 })
|
||||
merged("novox/mesh-catalog", 301, catalogMerge, delivery.Forge{Root: "r0",
|
||||
merged("novox/mesh-catalog", 301, catalogMerge, forgemsg.Forge{Root: "r0",
|
||||
Trees: map[string]string{"modules/importer": "t1", "modules/plain": "t2"},
|
||||
Manifests: map[string]string{"modules/importer": named("importer"), "modules/plain": named("plain")},
|
||||
Locks: map[string]map[string]string{"modules/importer": {"go.sum": goSumHash}, "modules/plain": {"go.sum": goSumHash}}})
|
||||
@@ -140,11 +141,11 @@ func TestAMeshSDKMergeMovesOnlyTheModulesImportingAChangedPackage(t *testing.T)
|
||||
// over its named tree, is not written (issue 1758, ADR 0499 clause 6; the operator, 2026-10-25: a module's value moves
|
||||
// only with the module).
|
||||
const goOnly = "4444444444444444444444444444444444444444"
|
||||
merged("novox/mesh-sdk", 72, goOnly, delivery.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc",
|
||||
merged("novox/mesh-sdk", 72, goOnly, forgemsg.Forge{Root: "cccccccccccccccccccccccccccccccccccccccc",
|
||||
Trees: map[string]string{"": "cccccccccccccccccccccccccccccccccccccccc"}, Manifests: map[string]string{"": string(sdkManifest)},
|
||||
Paths: sdkObjects("6666666666666666666666666666666666666666")})
|
||||
// mesh-sdk merges a change of buildstep alone (its version moves: src changed too).
|
||||
merged("novox/mesh-sdk", 71, m2, delivery.Forge{Root: sdk2, Trees: map[string]string{"": sdk2},
|
||||
merged("novox/mesh-sdk", 71, m2, forgemsg.Forge{Root: sdk2, Trees: map[string]string{"": sdk2},
|
||||
Manifests: map[string]string{"": string(sdkManifest)}, Paths: sdkObjects("7777777777777777777777777777777777777777")})
|
||||
sdkAt2 := module("mesh-sdk", func(m delivery.Module) bool { return m.Commit == m2 })
|
||||
if sdkAt2.Inputs2 == nil || !strings.HasPrefix(sdkAt2.Inputs2.Tree, delivery.NamedTreePrefix) {
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
@@ -73,7 +74,10 @@ func (w *previewWatch) ensure() error {
|
||||
}
|
||||
})
|
||||
}()
|
||||
return w.ix.RecomputeCandidates(w.ctx)
|
||||
if err := w.ix.RecomputeCandidates(w.ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return w.ix.WriteTryCandidates(w.ctx)
|
||||
}
|
||||
|
||||
// Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the
|
||||
@@ -90,3 +94,44 @@ func (w *previewWatch) Written(m delivered) {
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
// tryEventSubject is the subject of an event of the checker's stream checks about a try.
|
||||
func tryEventSubject(event string) string {
|
||||
return "mesh.stream." + delivery.CheckerModule + ".event." + event
|
||||
}
|
||||
|
||||
// heardChecks takes one event of the checker's stream in: a preview written (Written), or a whole try's start or end, from
|
||||
// which the candidate of its group is written or deleted (index.Indexer.StartTry, EndTry). A failure stays unacknowledged, and
|
||||
// the server delivers it again; a body that is no try event is terminated.
|
||||
func (w *previewWatch) heardChecks(m delivered) {
|
||||
var event string
|
||||
switch m.Subject() {
|
||||
case tryEventSubject(tryevent.Started):
|
||||
event = tryevent.Started
|
||||
case tryEventSubject(tryevent.Ended):
|
||||
event = tryevent.Ended
|
||||
default:
|
||||
w.Written(m)
|
||||
return
|
||||
}
|
||||
e, err := tryevent.Decode(event, m.Data())
|
||||
if err != nil {
|
||||
w.logger.Printf("%s: not a try event, dropped: %v", m.Subject(), err)
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if event == tryevent.Started {
|
||||
var ms []index.TryMember
|
||||
for _, x := range e.Members {
|
||||
ms = append(ms, index.TryMember{Repository: x.Repository, Head: x.Head, Trunk: x.Trunk})
|
||||
}
|
||||
err = w.ix.StartTry(w.ctx, e.ID, ms)
|
||||
} else {
|
||||
err = w.ix.EndTry(w.ctx, e.ID)
|
||||
}
|
||||
if err != nil {
|
||||
w.logger.Printf("%s of %s: left for redelivery: %v", event, e.ID, err)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
@@ -16,6 +16,11 @@ type takerIn interface {
|
||||
TakeIn(ctx context.Context, module string) index.TakenIn
|
||||
TakeInRepository(ctx context.Context, repository string) index.TakenIn
|
||||
JudgeAgain(ctx context.Context, module, digest string) index.TakenIn
|
||||
TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn
|
||||
DropPending(ctx context.Context, key, by, why string) index.TakenIn
|
||||
TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn
|
||||
DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn
|
||||
RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn
|
||||
}
|
||||
|
||||
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
|
||||
@@ -34,7 +39,22 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
|
||||
return
|
||||
}
|
||||
asked, r := "take-in", index.TakenIn{}
|
||||
if a.Repository != "" {
|
||||
if a.RecoverTries {
|
||||
asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why)
|
||||
a.Module = "tries"
|
||||
} else if a.TryCandidate != "" && a.Drop {
|
||||
asked, r = "drop of a try's candidate", ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why)
|
||||
a.Module = a.TryCandidate
|
||||
} else if a.Merge != "" {
|
||||
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
|
||||
a.Module = r.Module
|
||||
} else if a.Pending != "" && a.Drop {
|
||||
asked, r = "drop of a pending merge", ix.DropPending(ctx, a.Pending, a.By, a.Why)
|
||||
a.Module = a.Pending
|
||||
} else if a.Pending != "" {
|
||||
asked, r = "hand take-in of what waits", ix.TakeInPendingByHand(ctx, a.Pending, a.By, a.Why)
|
||||
a.Module = a.Pending
|
||||
} else if a.Repository != "" {
|
||||
asked, r = "take-in of a repository", ix.TakeInRepository(ctx, a.Repository)
|
||||
a.Module = a.Repository
|
||||
} else if a.Judge {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"github.com/nats-io/nats.go"
|
||||
@@ -62,3 +63,28 @@ func sayAccepted(js jetstream.JetStream, cred process.Account, logf func(string,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sayTakeIn says a take-in event (novox/hq ADR 0633 §5) as the indexer's ordinary event, on the path its other events
|
||||
// take: mesh.mod.<module>.event.take-in.<kind>, the wait's key in the body. A publish that fails is the caller's to say.
|
||||
func sayTakeIn(js jetstream.JetStream, cred process.Account) func(context.Context, takein.TakeIn) error {
|
||||
return func(ctx context.Context, t takein.TakeIn) error {
|
||||
event, err := takein.TakeInEvent(t.Kind)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := takein.EncodeTakeIn(t)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := nats.NewMsg(cred.EventSubject(event))
|
||||
msg.Header.Set("x-source", cred.Module)
|
||||
msg.Header.Set("x-node", cred.Node)
|
||||
msg.Header.Set("x-time", time.Now().UTC().Format(time.RFC3339Nano))
|
||||
msg.Header.Set("content-type", "application/json")
|
||||
msg.Data = data
|
||||
pub, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
_, err = js.PublishMsg(pub, msg)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,7 +46,19 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintln(stderr, "mesh-judge:", err)
|
||||
return 1
|
||||
}
|
||||
v, err := judge.Request(r)
|
||||
// The request's seat definitions (ADR 0636 §4): the composer's request type of this build does not decode them.
|
||||
var seats struct {
|
||||
Seats []json.RawMessage `json:"seats"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &seats); err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-judge:", err)
|
||||
return 1
|
||||
}
|
||||
files := make([][]byte, len(seats.Seats))
|
||||
for i, s := range seats.Seats {
|
||||
files[i] = s
|
||||
}
|
||||
v, err := judge.RequestWith(r, files)
|
||||
if err != nil {
|
||||
fmt.Fprintln(stderr, "mesh-judge:", err)
|
||||
return 1
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
@@ -24,6 +24,14 @@ require (
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun
|
||||
|
||||
require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
|
||||
require (
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein v0.0.0-00010101000000-000000000000
|
||||
git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000
|
||||
)
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein => ./takein
|
||||
|
||||
replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ../mesh-delivery-forge-bridge/forgemsg
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659 h1:tK7MzxVm/ECNg1Yblhh1ynVEthbZCg2mSPW0wQZC0b0=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261104002338-5a38a1221659/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -7,12 +7,13 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func manifestPull(t *testing.T, number int, head, manifest string) delivery.Forge {
|
||||
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
|
||||
func manifestPull(t *testing.T, number int, head, manifest string) forgemsg.Forge {
|
||||
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
|
||||
Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
|
||||
if manifest != "" {
|
||||
f.Manifests = map[string]string{"modules/avahi": manifest}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,9 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -80,6 +83,9 @@ type Indexer struct {
|
||||
// repository it replaces by a directory (fingerprint.ReplacedSiblings), which the version line takes by their
|
||||
// content; nil reads none.
|
||||
GoMod func(ctx context.Context, repository, sha, dir string) ([]byte, error)
|
||||
// ReadFile reads one file of a repository at a commit (gitea.gitea_get_file); a file the commit does not hold is nil.
|
||||
// With Files it reads the seat definitions a revision carries (seatsof.go); nil reads none.
|
||||
ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error)
|
||||
// Now is the clock of a taken value; nil is time.Now.
|
||||
Now func() time.Time
|
||||
Builds *Builds
|
||||
@@ -98,7 +104,7 @@ type Indexer struct {
|
||||
// ReadOpen asks the forge which pull requests into the trunk are open now (gitea_open_pulls). A catalogue pull message
|
||||
// that finds the candidates of another pull request asks it and deletes those of a pull request no longer open, so a
|
||||
// closed message the indexer missed heals at the next one (issue 1126); nil asks none.
|
||||
ReadOpen func(ctx context.Context) ([]delivery.OpenPull, error)
|
||||
ReadOpen func(ctx context.Context) ([]forgemsg.OpenPull, error)
|
||||
|
||||
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
@@ -106,7 +112,7 @@ type Indexer struct {
|
||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||
tried map[string]bool // repository, commit and directory a registered module's manifest was tried at (registered.go)
|
||||
unread map[string]bool // registered modules whose last read the forge could not answer, read again by RetryUnread
|
||||
moving map[string]delivery.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
moving map[string]forgemsg.Forge // by modules key: a merge naming a module another repository holds, waiting for that trunk to drop it (moves.go)
|
||||
retrying atomic.Bool // a RetryUnread runs in the background (kickRetry)
|
||||
retryAgain atomic.Bool // an event asked for one more RetryUnread after the running one
|
||||
retryWG sync.WaitGroup // the background RetryUnread, for tests to wait on
|
||||
@@ -115,12 +121,38 @@ type Indexer struct {
|
||||
filling atomic.Bool // a FillTaken runs in the background (KickFill)
|
||||
fillAgain atomic.Bool // a kick arrived while it ran
|
||||
atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished)
|
||||
saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head
|
||||
failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes
|
||||
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
|
||||
OwnBuild func(ctx context.Context) (string, error)
|
||||
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
|
||||
ownMu sync.Mutex // guards ownFP
|
||||
ownFP string // the fingerprint of the build this program runs, asked at the start pass (learnOwnBuild)
|
||||
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
||||
OwnBuild func(ctx context.Context) (string, error)
|
||||
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
|
||||
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
|
||||
ownMu sync.Mutex // guards ownFP
|
||||
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
|
||||
ownFixed bool // FixOwnBuild asked already: it asks once
|
||||
ownSince time.Time // when the build was asked and not known, with ownWhy
|
||||
ownWhy string
|
||||
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
|
||||
OwnBuildState Store
|
||||
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
|
||||
// Message asks the forge bridge for the merged message of a merge, repository and merge commit in, the message whole out (a
|
||||
// pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none.
|
||||
Message func(ctx context.Context, repository, merge string) ([]byte, error)
|
||||
heldMu sync.Mutex
|
||||
held map[uint64]forgemsg.Forge // the merged messages Arrive was handed, by stream sequence, until their merge is taken in or dropped
|
||||
// Pending is the state of merges held for the indexer their revision names (pending.go, novox/hq ADR 0633 §1); nil
|
||||
// takes every merge in at once, as before the take-in step.
|
||||
Pending Store
|
||||
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
|
||||
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
|
||||
// OpenTries lists the tries that wait or run, from the checker's tool (checker_tries); nil recovers none (trycandidates.go).
|
||||
OpenTries func(ctx context.Context) ([]tryevent.Open, error)
|
||||
// TryRecovery keeps how the last recovery of the open tries went (trycandidates.go); nil keeps none.
|
||||
TryRecovery Store
|
||||
tryMu sync.Mutex // one try event or recovery at a time (trycandidates.go)
|
||||
recoveryFailed atomic.Bool // the last recovery of the open tries failed (trycandidates.go)
|
||||
tries map[string][]TryMember // the whole tries that wait or run, by try id (trycandidates.go); guarded by mu
|
||||
pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go)
|
||||
}
|
||||
|
||||
func (ix *Indexer) logf(format string, args ...any) {
|
||||
@@ -133,23 +165,23 @@ func (ix *Indexer) logf(format string, args ...any) {
|
||||
// pull request, a closed one or a merged message takes it out; a group left with none is deleted. A change is written
|
||||
// by compare-and-set on the revision read, and read again on a conflict. A lone pull request is a group of one. A
|
||||
// merged message also writes the fingerprint of each module of its repository (modules.go).
|
||||
func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
func (ix *Indexer) Apply(ctx context.Context, f forgemsg.Forge) error {
|
||||
_, err := ix.ApplyMessage(ctx, f)
|
||||
return err
|
||||
}
|
||||
|
||||
// ApplyMessage is Apply, and answers which modules a merged message wrote and kept (for the indexer's heard state). Each
|
||||
// applied merged message is said in one line: its repository, commit and pull request, and the modules written.
|
||||
func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied, error) {
|
||||
func (ix *Indexer) ApplyMessage(ctx context.Context, f forgemsg.Forge) (Applied, error) {
|
||||
ix.kickJudge(ctx)
|
||||
var applied Applied
|
||||
if f.Kind == delivery.ForgeOpen {
|
||||
if f.Kind == forgemsg.ForgeOpen {
|
||||
return applied, ix.applyOpen(ctx, f)
|
||||
}
|
||||
if f.Kind != delivery.ForgePull && f.Kind != delivery.ForgeMerged {
|
||||
if f.Kind != forgemsg.ForgePull && f.Kind != forgemsg.ForgeMerged {
|
||||
return applied, nil
|
||||
}
|
||||
if f.Kind == delivery.ForgeMerged && ix.Modules != nil {
|
||||
if f.Kind == forgemsg.ForgeMerged && ix.Modules != nil {
|
||||
w, err := ix.applyMerged(ctx, f)
|
||||
applied = Applied{Written: w.written, Late: w.late}
|
||||
if err != nil {
|
||||
@@ -178,7 +210,7 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied,
|
||||
}
|
||||
var digests map[string]string
|
||||
var held map[string]delivery.Module
|
||||
if f.Kind == delivery.ForgePull {
|
||||
if f.Kind == forgemsg.ForgePull {
|
||||
var err error
|
||||
if digests, err = ix.keepManifests(ctx, f); err != nil {
|
||||
return applied, err
|
||||
@@ -207,15 +239,26 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied,
|
||||
ix.logf("%s#%d: its config declarations are not written: %v", f.Repository, f.Number, err)
|
||||
}
|
||||
}
|
||||
return applied, ix.writeCandidate(ctx, f, digests, held)
|
||||
}
|
||||
|
||||
// removeCandidate takes a merged pull request out of its group's candidate and writes nothing else of its message: for a
|
||||
// merge a newer merge of the same repository covers (it needs only the repository, number and branch).
|
||||
func (ix *Indexer) removeCandidate(ctx context.Context, f forgemsg.Forge) error {
|
||||
return ix.writeCandidate(ctx, f, nil, nil)
|
||||
}
|
||||
|
||||
// writeCandidate writes the group's candidate with the pull request f names (taken out for a merged or closed one).
|
||||
func (ix *Indexer) writeCandidate(ctx context.Context, f forgemsg.Forge, digests map[string]string, held map[string]delivery.Module) error {
|
||||
if trunkLike[f.Branch] {
|
||||
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
|
||||
return applied, nil
|
||||
return nil
|
||||
}
|
||||
key, err := delivery.CandidatesKey(f.Branch)
|
||||
if err != nil {
|
||||
return applied, fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
|
||||
return fmt.Errorf("%w: %s#%d: %v", ErrUnreadable, f.Repository, f.Number, err)
|
||||
}
|
||||
return applied, write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
return write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
var cur *delivery.Candidate
|
||||
if found {
|
||||
c, err := delivery.DecodeCandidate(value)
|
||||
@@ -245,8 +288,8 @@ func (ix *Indexer) ApplyMessage(ctx context.Context, f delivery.Forge) (Applied,
|
||||
// applyOpen takes an open message into the candidates (novox/hq ADR 0443 §5): every pull request it does not name is
|
||||
// taken out of its group, and a group left with none is deleted. What the message names is left as it is. A pull request
|
||||
// opened after the bridge read the forge is heard after this message and added again by its own.
|
||||
func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
|
||||
named := make(map[delivery.OpenPull]bool, len(f.Open))
|
||||
func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error {
|
||||
named := make(map[forgemsg.OpenPull]bool, len(f.Open))
|
||||
for _, p := range f.Open {
|
||||
named[p] = true
|
||||
}
|
||||
@@ -255,6 +298,9 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
|
||||
return fmt.Errorf("listing the candidates: %w", err)
|
||||
}
|
||||
for _, key := range keys {
|
||||
if IsTryGroup(key) {
|
||||
continue // a try's candidate is no pull request's: the checker's try owns it (trycandidates.go)
|
||||
}
|
||||
err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
@@ -264,7 +310,7 @@ func (ix *Indexer) applyOpen(ctx context.Context, f delivery.Forge) error {
|
||||
return nil, false, fmt.Errorf("candidates.%s holds what is not a candidate: %w", key, err)
|
||||
}
|
||||
kept := slices.DeleteFunc(slices.Clone(c.Pulls), func(p delivery.CandidatePull) bool {
|
||||
return !named[delivery.OpenPull{Repository: p.Repository, Number: p.Number}]
|
||||
return !named[forgemsg.OpenPull{Repository: p.Repository, Number: p.Number}]
|
||||
})
|
||||
switch {
|
||||
case len(kept) == len(c.Pulls):
|
||||
@@ -331,14 +377,14 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte,
|
||||
// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a
|
||||
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
|
||||
// directory; held are the modules the modules bucket holds, which name the module of a changed directory (moduleOfDir).
|
||||
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate {
|
||||
func (ix *Indexer) next(cur *delivery.Candidate, f forgemsg.Forge, digests map[string]string, held map[string]delivery.Module) *delivery.Candidate {
|
||||
var pulls []delivery.CandidatePull
|
||||
var base uint64
|
||||
if cur != nil {
|
||||
pulls, base = slices.Clone(cur.Pulls), cur.Base
|
||||
}
|
||||
at := slices.IndexFunc(pulls, func(p delivery.CandidatePull) bool { return p.Repository == f.Repository && p.Number == f.Number })
|
||||
if f.Kind == delivery.ForgeMerged || f.Action == delivery.PullClosed {
|
||||
if f.Kind == forgemsg.ForgeMerged || f.Action == forgemsg.PullClosed {
|
||||
if at < 0 {
|
||||
return cur
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -27,7 +28,7 @@ var (
|
||||
|
||||
// writePredefined keeps the predefined list a merged message carries, unless the key names a later merge: a merge older
|
||||
// than the one it names never replaces it (ADR 0348 §7). The same list at the same merge writes nothing (ADR 0289 §1).
|
||||
func (ix *Indexer) writePredefined(ctx context.Context, f delivery.Forge) error {
|
||||
func (ix *Indexer) writePredefined(ctx context.Context, f forgemsg.Forge) error {
|
||||
if ix.Predefined == nil || len(f.Predefined) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -203,8 +204,8 @@ func (ix *Indexer) dropConfigCandidate(ctx context.Context, repository string, n
|
||||
|
||||
// configCandidateOfPull is a pull message's part in config-candidates: a head whose manifests the message carries is written,
|
||||
// a closed one deleted. A head held again without its directories leaves the key as it is.
|
||||
func (ix *Indexer) configCandidateOfPull(ctx context.Context, f delivery.Forge, held map[string]delivery.Module) error {
|
||||
if f.Action == delivery.PullClosed {
|
||||
func (ix *Indexer) configCandidateOfPull(ctx context.Context, f forgemsg.Forge, held map[string]delivery.Module) error {
|
||||
if f.Action == forgemsg.PullClosed {
|
||||
return ix.dropConfigCandidate(ctx, f.Repository, f.Number)
|
||||
}
|
||||
if !f.DirsKnown {
|
||||
@@ -220,7 +221,7 @@ func (ix *Indexer) configCandidateOfPull(ctx context.Context, f delivery.Forge,
|
||||
// configCandidatesAfterMerge writes again every open pull request's key that names a module the merge changed, since what
|
||||
// it adds is judged against the trunk (ADR 0565 §2), from the head's manifests its candidate keeps; and deletes the merged
|
||||
// pull request's.
|
||||
func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f delivery.Forge, changed []string) error {
|
||||
func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f forgemsg.Forge, changed []string) error {
|
||||
if ix.ConfigCandidates == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -243,6 +244,9 @@ func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f delivery.Fo
|
||||
}
|
||||
var errs []error
|
||||
for _, g := range groups {
|
||||
if IsTryGroup(g) {
|
||||
continue // a try's candidate holds no pull request's config declarations (trycandidates.go)
|
||||
}
|
||||
raw, _, found, err := ix.Store.Get(ctx, g)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -77,12 +78,29 @@ func TestTheControlRecords(t *testing.T) {
|
||||
t.Errorf("a head declaring nothing new = %+v, want an empty set", c)
|
||||
}
|
||||
closed := pull(catalogueRepo, 7, "feat/y", strings.Repeat("d", 40))
|
||||
closed.Action = delivery.PullClosed
|
||||
closed.Action = forgemsg.PullClosed
|
||||
if err := ix.Apply(ctx, closed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := cands.vals[key]; ok {
|
||||
t.Error("a closed pull request's declarations stayed")
|
||||
}
|
||||
// A group try's candidate (number 0) holds no pull request's declarations: a merge that touches its module writes none
|
||||
// for it (fails when configCandidatesAfterMerge no longer skips a try's group).
|
||||
tryPull := delivery.CandidatePull{Head: delivery.Head{Repository: catalogueRepo, Head: h1}, Modules: []string{"avahi"},
|
||||
Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(trunk)), None2: NoneNoPreview}}}
|
||||
tryCand := delivery.Candidate{V: delivery.Version, Group: "try-5", Pulls: []delivery.CandidatePull{tryPull}}
|
||||
tryCand.HeadsHash, _ = delivery.HeadsHash([]delivery.Head{tryPull.Head})
|
||||
tryCand.FPSet = delivery.FingerprintSet(tryCand.Pulls)
|
||||
data, err := delivery.EncodeCandidate(tryCand)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Store.Create(ctx, "try-5", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.configCandidatesAfterMerge(ctx, mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t2"}), []string{"avahi"}); err != nil || len(cands.vals) != 0 {
|
||||
t.Errorf("a merge touching a try's module wrote %v for it (%v), want none", cands.vals, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2,9 +2,11 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -25,6 +27,16 @@ type memStore struct {
|
||||
next uint64
|
||||
writes int
|
||||
beforeWrite func()
|
||||
failWrite func(key string) error // a write's own error, asked before each Create and Update
|
||||
maxValue int // the bucket's value cap (buscompose.StateMaxValueBytes); a larger value is refused as the bus refuses it
|
||||
}
|
||||
|
||||
// tooBig is the bus's refusal of a value over a bucket's cap.
|
||||
func (m *memStore) tooBig(value []byte) error {
|
||||
if m.maxValue > 0 && len(value) > m.maxValue {
|
||||
return errors.New("nats: API error: code=400 err_code=10054 description=message size exceeds maximum allowed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
@@ -58,18 +70,34 @@ func (m *memStore) put(key string, value []byte) {
|
||||
}
|
||||
func (m *memStore) Create(_ context.Context, key string, value []byte) error {
|
||||
m.hook()
|
||||
if f := m.failWrite; f != nil {
|
||||
if err := f(key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, ok := m.vals[key]; ok {
|
||||
return ErrConflict
|
||||
}
|
||||
if err := m.tooBig(value); err != nil {
|
||||
return err
|
||||
}
|
||||
m.writes++
|
||||
m.put(key, value)
|
||||
return nil
|
||||
}
|
||||
func (m *memStore) Update(_ context.Context, key string, value []byte, rev uint64) error {
|
||||
m.hook()
|
||||
if f := m.failWrite; f != nil {
|
||||
if err := f(key); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, ok := m.vals[key]; !ok || m.revs[key] != rev {
|
||||
return ErrConflict
|
||||
}
|
||||
if err := m.tooBig(value); err != nil {
|
||||
return err
|
||||
}
|
||||
m.writes++
|
||||
m.put(key, value)
|
||||
return nil
|
||||
@@ -91,9 +119,9 @@ func buildRecord(t *testing.T, repo, module, path, at string) []byte {
|
||||
return logged(t, "build-"+module, module, repo, path, "", "", at)
|
||||
}
|
||||
|
||||
func pull(repo string, number int, branch, head string, dirs ...string) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head,
|
||||
Action: delivery.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil}
|
||||
func pull(repo string, number int, branch, head string, dirs ...string) forgemsg.Forge {
|
||||
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: repo, Branch: branch, Number: number, Head: head,
|
||||
Action: forgemsg.PullUpdated, Dirs: dirs, DirsKnown: dirs != nil}
|
||||
}
|
||||
|
||||
func newIndexer(t *testing.T) (*Indexer, *memStore, *strings.Builder) {
|
||||
@@ -212,13 +240,13 @@ func TestClosedAndMergedTakeAPullOutAndTheLastDeletesTheKey(t *testing.T) { // 1
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
_ = ix.Apply(ctx, pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"))
|
||||
gone := func(repo string, n int, head string) delivery.Forge {
|
||||
gone := func(repo string, n int, head string) forgemsg.Forge {
|
||||
if how == "closed" {
|
||||
f := pull(repo, n, "feat/x", head)
|
||||
f.Action = delivery.PullClosed
|
||||
f.Action = forgemsg.PullClosed
|
||||
return f
|
||||
}
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
|
||||
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: n, Head: head, Merge: h3, Root: "r", Trees: map[string]string{"a": "t"}}
|
||||
}
|
||||
if err := ix.Apply(ctx, gone("novox/mesh-delivery", 12, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -330,8 +358,8 @@ func TestAnOddBranchNameKeysEscaped(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func open(pulls ...delivery.OpenPull) delivery.Forge {
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeOpen, Open: pulls}
|
||||
func open(pulls ...forgemsg.OpenPull) forgemsg.Forge {
|
||||
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeOpen, Open: pulls}
|
||||
}
|
||||
|
||||
// 1279/AC7 (ADR 0443 §5): on an open message, candidates lose every pull request it does not name, a group left empty is
|
||||
@@ -339,7 +367,7 @@ func open(pulls ...delivery.OpenPull) delivery.Forge {
|
||||
func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/AC7
|
||||
ix, st, _ := newIndexer(t)
|
||||
ctx := context.Background()
|
||||
for _, f := range []delivery.Forge{
|
||||
for _, f := range []forgemsg.Forge{
|
||||
pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"), pull("novox/mesh-catalog", 300, "feat/x", h2, "modules/c"),
|
||||
pull("novox/mesh-delivery", 13, "feat/y", h2, "modules/a"),
|
||||
pull("novox/mesh-catalog", 280, "feat/old", h3, "modules/c"),
|
||||
@@ -351,7 +379,7 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A
|
||||
before, _ := read(t, st, "feat/x")
|
||||
|
||||
// Open: mesh-delivery#12 only (and a pull request no group holds, which changes nothing).
|
||||
if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 999})); err != nil {
|
||||
if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 999})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c, ok := read(t, st, "feat/x")
|
||||
@@ -374,7 +402,7 @@ func TestAnOpenMessageDropsEveryCandidateItDoesNotName(t *testing.T) { // 1279/A
|
||||
}
|
||||
// The same open message again changes nothing (no write), and an empty one drops every group.
|
||||
writes := st.writes
|
||||
if err := ix.Apply(ctx, open(delivery.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, delivery.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil {
|
||||
if err := ix.Apply(ctx, open(forgemsg.OpenPull{Repository: "novox/mesh-delivery", Number: 12}, forgemsg.OpenPull{Repository: "novox/mesh-catalog", Number: 300})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.writes != writes {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -48,7 +49,7 @@ func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string
|
||||
|
||||
// keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory.
|
||||
// A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest.
|
||||
func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) {
|
||||
func (ix *Indexer) keepManifests(ctx context.Context, f forgemsg.Forge) (map[string]string, error) {
|
||||
digests := make(map[string]string, len(f.Manifests))
|
||||
for dir, raw := range f.Manifests {
|
||||
module := manifestModule(raw) // the name the manifest gives itself: one digest always has one module
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -37,7 +38,7 @@ func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memSt
|
||||
return ix, mods, manifests
|
||||
}
|
||||
|
||||
func withManifests(f delivery.Forge, manifests map[string]string, locks map[string]map[string]string) delivery.Forge {
|
||||
func withManifests(f forgemsg.Forge, manifests map[string]string, locks map[string]map[string]string) forgemsg.Forge {
|
||||
f.Manifests, f.Locks = manifests, locks
|
||||
return f
|
||||
}
|
||||
@@ -81,8 +82,8 @@ func sameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Errorf("a second merge: %v", err)
|
||||
}
|
||||
pull := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
|
||||
pull := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
|
||||
Action: forgemsg.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
|
||||
if err := ix.Apply(context.Background(), pull); err != nil {
|
||||
t.Errorf("a pull message: %v", err)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
@@ -31,20 +32,20 @@ import (
|
||||
//
|
||||
// A merge never moves a module backwards (order.go): a merged message older than the repository's taken merge writes and
|
||||
// deletes nothing, and is said; it is asked once, before anything is written.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) (*mergeWrite, error) {
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f forgemsg.Forge) (*mergeWrite, error) {
|
||||
return ix.applyManifests(ctx, f, true)
|
||||
}
|
||||
|
||||
// applyManifests is applyMerged; drop says whether the modules of the repository the message holds no manifest for are
|
||||
// deleted. A message that names only some of a repository's modules (a registered module read on its own, registered.go)
|
||||
// drops none.
|
||||
func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bool) (*mergeWrite, error) {
|
||||
func (ix *Indexer) applyManifests(ctx context.Context, f forgemsg.Forge, drop bool) (*mergeWrite, error) {
|
||||
return ix.applyMoving(ctx, f, drop, nil)
|
||||
}
|
||||
|
||||
// applyMoving is applyManifests; moves names the modules keys the message may take from the repository that holds them
|
||||
// (a move handed over, moves.go). Every other module held of another repository is left, said and kept waiting.
|
||||
func (ix *Indexer) applyMoving(ctx context.Context, f delivery.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) {
|
||||
func (ix *Indexer) applyMoving(ctx context.Context, f forgemsg.Forge, drop bool, moves map[string]bool) (*mergeWrite, error) {
|
||||
w := newMergeWrite(f)
|
||||
if drop {
|
||||
if err := ix.lateMerge(ctx, w); err != nil || w.late != "" {
|
||||
@@ -180,7 +181,7 @@ func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m
|
||||
// namedObjects are, by module directory, the git objects of the paths each manifest of the merge names for every artifact
|
||||
// (catalogue Manifest.NamedPaths): those the message carries, else asked of the forge at the merge commit in one call
|
||||
// (Named). A forge that cannot say gives, for each directory asked, its words, so the module's reason names them.
|
||||
func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[string]map[string]string, map[string]string) {
|
||||
func (ix *Indexer) namedObjects(ctx context.Context, f forgemsg.Forge) (map[string]map[string]string, map[string]string) {
|
||||
objects := map[string]map[string]string{}
|
||||
need := map[string][]string{}
|
||||
for dir, raw := range f.Manifests {
|
||||
@@ -220,7 +221,7 @@ func (ix *Indexer) namedObjects(ctx context.Context, f delivery.Forge) (map[stri
|
||||
// held(key, module) says whether the trunk holds it. A module of another repository, or of none known, is never touched.
|
||||
func (ix *Indexer) dropRemoved(ctx context.Context, repository string, held map[string]delivery.Module, trunkHolds func(key string, m delivery.Module) bool) error {
|
||||
var handed []string
|
||||
elsewhere := map[string]delivery.Forge{}
|
||||
elsewhere := map[string]forgemsg.Forge{}
|
||||
for _, key := range slices.Sorted(maps.Keys(held)) {
|
||||
cur := held[key]
|
||||
if cur.Repository == "" || !delivery.SameRepository(cur.Repository, repository) || trunkHolds(key, cur) {
|
||||
@@ -354,7 +355,7 @@ func unreadInputs(tree string, locks, paths map[string]string, packages []string
|
||||
// goSumPackages are the version line's packages of the merge's directory dir (ADR 0573 §2, ADR 0593 §2): the mesh Go
|
||||
// modules of other repositories its go.sum names, and those of its own repository its go.mod replaces by a directory
|
||||
// (siblingPackages), which no go.sum line names. A directory with no go.sum is no Go module and has none.
|
||||
func (ix *Indexer) goSumPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string {
|
||||
func (ix *Indexer) goSumPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string {
|
||||
others := ix.otherPackages(ctx, f, dir, cur, has)
|
||||
if f.Locks[dir]["go.sum"] == "" && len(f.Packages[dir]) == 0 {
|
||||
return others
|
||||
@@ -368,7 +369,7 @@ func (ix *Indexer) goSumPackages(ctx context.Context, f delivery.Forge, dir stri
|
||||
|
||||
// siblingPackages are the mesh Go modules of the merge's own repository the go.mod of dir replaces by a directory, read
|
||||
// from the forge (GoMod); where it cannot be read, the held module's own-repository packages stand, said.
|
||||
func (ix *Indexer) siblingPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string {
|
||||
func (ix *Indexer) siblingPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string {
|
||||
held := func() []string {
|
||||
if !has || cur.Inputs2 == nil {
|
||||
return nil
|
||||
@@ -395,7 +396,7 @@ func (ix *Indexer) siblingPackages(ctx context.Context, f delivery.Forge, dir st
|
||||
// otherPackages are the mesh Go modules of other repositories the go.sum of the merge's directory dir names: those the
|
||||
// message carries (an older bridge), else the held module's while its go.sum's hash is unchanged, else the go.sum read
|
||||
// from the forge (GoSum).
|
||||
func (ix *Indexer) otherPackages(ctx context.Context, f delivery.Forge, dir string, cur delivery.Module, has bool) []string {
|
||||
func (ix *Indexer) otherPackages(ctx context.Context, f forgemsg.Forge, dir string, cur delivery.Module, has bool) []string {
|
||||
if carried := f.Packages[dir]; len(carried) > 0 {
|
||||
return fingerprint.OtherPackages(carried, f.Repository) // its own repository's come with its tree
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -71,12 +72,12 @@ func rec(t *testing.T, id, module, repo, path, ref, state, at, inputs string) [2
|
||||
// mergedMsg is a merged message of the repository with these module directories' trees, each with a manifest naming its
|
||||
// module, the directory's last element ("r" for the root), as the forge bridge sends it (one manifest per directory that
|
||||
// holds a module.json).
|
||||
func mergedMsg(repo string, trees map[string]string) delivery.Forge {
|
||||
func mergedMsg(repo string, trees map[string]string) forgemsg.Forge {
|
||||
manifests := map[string]string{}
|
||||
for dir := range trees {
|
||||
manifests[dir] = `{"module":"` + nameOfDir(dir) + `","version":"1"}`
|
||||
}
|
||||
return delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests}
|
||||
return forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/x", Number: 12, Head: h1, Merge: merge, Root: "r0", Trees: trees, Manifests: manifests}
|
||||
}
|
||||
|
||||
func nameOfDir(dir string) string {
|
||||
@@ -459,7 +460,7 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
|
||||
gone.Trees, gone.Manifests = nil, nil
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
merges []delivery.Forge
|
||||
merges []forgemsg.Forge
|
||||
repository string // modules.a's repository at the end, "" for none
|
||||
deletedKeyA bool
|
||||
// Issue 1367: a fresh indexer, no wait in memory, the source naming heads; trunk is the module directories the forge
|
||||
@@ -472,20 +473,20 @@ func TestAModuleMovesBetweenRepositoriesOnlyWhenItsTrunkDropsIt(t *testing.T) {
|
||||
stored bool
|
||||
bGoneAtEnd bool
|
||||
}{
|
||||
{name: "named while its trunk holds it: left", merges: []delivery.Forge{held, named}, repository: from},
|
||||
{name: "named, then dropped by its trunk: handed over in place", merges: []delivery.Forge{held, named, dropped}, repository: to},
|
||||
{name: "dropped by its trunk, then named: deleted, then written new", merges: []delivery.Forge{held, dropped, named}, repository: to, deletedKeyA: true},
|
||||
{name: "dropped by its trunk with no merge naming it: deleted", merges: []delivery.Forge{held, dropped}, deletedKeyA: true},
|
||||
{name: "named while its trunk holds it: left", merges: []forgemsg.Forge{held, named}, repository: from},
|
||||
{name: "named, then dropped by its trunk: handed over in place", merges: []forgemsg.Forge{held, named, dropped}, repository: to},
|
||||
{name: "dropped by its trunk, then named: deleted, then written new", merges: []forgemsg.Forge{held, dropped, named}, repository: to, deletedKeyA: true},
|
||||
{name: "dropped by its trunk with no merge naming it: deleted", merges: []forgemsg.Forge{held, dropped}, deletedKeyA: true},
|
||||
{name: "no module.json left, another trunk holds it, nothing waits in memory: written from that trunk, never deleted",
|
||||
merges: []delivery.Forge{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true},
|
||||
merges: []forgemsg.Forge{held, gone}, heads: true, trunk: []string{"modules/a", "modules/other"}, repository: to, bGoneAtEnd: true},
|
||||
{name: "no module.json left, no other trunk holds it: deleted",
|
||||
merges: []delivery.Forge{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true},
|
||||
merges: []forgemsg.Forge{held, gone}, heads: true, deletedKeyA: true, bGoneAtEnd: true},
|
||||
{name: "taken in by repository, a head with no module.json: handed over from the other trunk",
|
||||
merges: []delivery.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true},
|
||||
merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: from, answer: "handed over", stored: true, repository: to, bGoneAtEnd: true},
|
||||
{name: "taken in by repository, a head with modules: answered by name",
|
||||
merges: []delivery.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from},
|
||||
merges: []forgemsg.Forge{held}, heads: true, trunk: []string{"modules/a"}, takeIn: to, answer: "by name", repository: from},
|
||||
{name: "taken in by repository, no head named: answered so",
|
||||
merges: []delivery.Forge{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from},
|
||||
merges: []forgemsg.Forge{held}, heads: true, takeIn: "novox/nobody", answer: "names no head", repository: from},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
mods := &deletesSeen{memStore: newMem()}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -14,9 +15,9 @@ import (
|
||||
// holds it at the head the latest source names: the module directories there are asked of the forge (Trunk) and the manifest of
|
||||
// each directory named like the module is read (Read). It answers the merge to hand the module over with, or none: a forge
|
||||
// that cannot say leaves the module to be dropped as it was, said (issue 1367).
|
||||
func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (delivery.Forge, bool) {
|
||||
func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository string) (forgemsg.Forge, bool) {
|
||||
if ix.Trunk == nil || ix.Read == nil {
|
||||
return delivery.Forge{}, false
|
||||
return forgemsg.Forge{}, false
|
||||
}
|
||||
ix.mu.Lock()
|
||||
heads := maps.Clone(ix.source.heads)
|
||||
@@ -45,7 +46,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st
|
||||
if read.Trees[dir] == "" || manifestModule(raw) != module {
|
||||
continue
|
||||
}
|
||||
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: other, Merge: head, Root: read.Root,
|
||||
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: other, Merge: head, Root: read.Root,
|
||||
Trees: map[string]string{dir: read.Trees[dir]}, Manifests: map[string]string{dir: raw}}
|
||||
if l := read.Locks[dir]; len(l) > 0 {
|
||||
f.Locks = map[string]map[string]string{dir: l}
|
||||
@@ -57,7 +58,7 @@ func (ix *Indexer) heldByAnotherTrunk(ctx context.Context, module, repository st
|
||||
return f, true
|
||||
}
|
||||
}
|
||||
return delivery.Forge{}, false
|
||||
return forgemsg.Forge{}, false
|
||||
}
|
||||
|
||||
// nameOfDirectory is the last element of a module directory.
|
||||
@@ -76,8 +77,8 @@ func nameOfDirectory(dir string) string { return dir[strings.LastIndex(dir, "/")
|
||||
|
||||
// waitToMove keeps, by modules key, the part of a merged message naming one module directory the indexer holds of another
|
||||
// repository. A later merge of the same repository replaces it.
|
||||
func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) {
|
||||
one := delivery.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root,
|
||||
func (ix *Indexer) waitToMove(key string, f forgemsg.Forge, dir string) {
|
||||
one := forgemsg.Forge{V: f.V, Kind: f.Kind, Repository: f.Repository, Branch: f.Branch, Number: f.Number, Merge: f.Merge, Root: f.Root,
|
||||
Trees: map[string]string{dir: f.Trees[dir]}, Manifests: map[string]string{dir: f.Manifests[dir]}}
|
||||
if l := f.Locks[dir]; len(l) > 0 {
|
||||
one.Locks = map[string]map[string]string{dir: l}
|
||||
@@ -91,7 +92,7 @@ func (ix *Indexer) waitToMove(key string, f delivery.Forge, dir string) {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
if ix.moving == nil {
|
||||
ix.moving = map[string]delivery.Forge{}
|
||||
ix.moving = map[string]forgemsg.Forge{}
|
||||
}
|
||||
ix.moving[key] = one
|
||||
}
|
||||
@@ -117,7 +118,7 @@ func (ix *Indexer) handOver(ctx context.Context, key string) error {
|
||||
}
|
||||
|
||||
// handOverFrom writes the module at key from the merge f, which names it, in place.
|
||||
func (ix *Indexer) handOverFrom(ctx context.Context, key string, f delivery.Forge) error {
|
||||
func (ix *Indexer) handOverFrom(ctx context.Context, key string, f forgemsg.Forge) error {
|
||||
w, err := ix.applyMoving(ctx, f, false, map[string]bool{key: true})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s, moving to %s: %w", key, f.Repository, err)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -46,7 +47,7 @@ func (w *mergeWrite) taken(digest string) {
|
||||
w.fresh[digest] = true
|
||||
}
|
||||
|
||||
func newMergeWrite(f delivery.Forge) *mergeWrite {
|
||||
func newMergeWrite(f forgemsg.Forge) *mergeWrite {
|
||||
return &mergeWrite{repository: f.Repository, merge: f.Merge}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// overlayStore is a store whose writes stay in memory over the store it reads through (novox/hq ADR 0633 §1): a merge
|
||||
// applied to it is computed whole, its every fingerprint and verdict, and nothing reaches the bus. A key it wrote reads
|
||||
// as written, a key it deleted as absent, every other as the store under it holds it. Its compare-and-set holds against
|
||||
// what it reads: a key it wrote has a revision of its own, counted above every revision under it. A nil store under
|
||||
// it holds nothing.
|
||||
type overlayStore struct {
|
||||
under Store
|
||||
mu sync.Mutex
|
||||
vals map[string]overlaid
|
||||
next uint64
|
||||
}
|
||||
|
||||
type overlaid struct {
|
||||
value []byte
|
||||
revision uint64
|
||||
deleted bool
|
||||
}
|
||||
|
||||
func newOverlay(under Store) *overlayStore {
|
||||
return &overlayStore{under: under, vals: map[string]overlaid{}, next: 1 << 62}
|
||||
}
|
||||
|
||||
func (o *overlayStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
o.mu.Lock()
|
||||
v, ok := o.vals[key]
|
||||
o.mu.Unlock()
|
||||
if ok {
|
||||
if v.deleted {
|
||||
return nil, v.revision, false, nil
|
||||
}
|
||||
return slices.Clone(v.value), v.revision, true, nil
|
||||
}
|
||||
if o.under == nil {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
return o.under.Get(ctx, key)
|
||||
}
|
||||
|
||||
// put checks the revision read against what the key holds now and writes it.
|
||||
func (o *overlayStore) put(ctx context.Context, key string, value []byte, revision uint64, create, del bool) error {
|
||||
_, rev, found, err := o.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
switch {
|
||||
case create && found, !create && (!found || rev != revision):
|
||||
return ErrConflict
|
||||
}
|
||||
o.next++
|
||||
o.vals[key] = overlaid{value: slices.Clone(value), revision: o.next, deleted: del}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *overlayStore) Create(ctx context.Context, key string, value []byte) error {
|
||||
return o.put(ctx, key, value, 0, true, false)
|
||||
}
|
||||
|
||||
func (o *overlayStore) Update(ctx context.Context, key string, value []byte, revision uint64) error {
|
||||
return o.put(ctx, key, value, revision, false, false)
|
||||
}
|
||||
|
||||
func (o *overlayStore) Delete(ctx context.Context, key string, revision uint64) error {
|
||||
return o.put(ctx, key, nil, revision, false, true)
|
||||
}
|
||||
|
||||
func (o *overlayStore) Keys(ctx context.Context) ([]string, error) {
|
||||
var keys []string
|
||||
if o.under != nil {
|
||||
var err error
|
||||
if keys, err = o.under.Keys(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
keys = slices.DeleteFunc(keys, func(k string) bool { _, ok := o.vals[k]; return ok })
|
||||
for k, v := range o.vals {
|
||||
if !v.deleted {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// written is what the overlay holds of key: its value, and whether it wrote or deleted it at all.
|
||||
func (o *overlayStore) written(key string) (value []byte, deleted, ok bool) {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
v, ok := o.vals[key]
|
||||
return slices.Clone(v.value), v.deleted, ok
|
||||
}
|
||||
|
||||
// changed is every key the overlay wrote or deleted, in order.
|
||||
func (o *overlayStore) changed() []string {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
return slices.Sorted(maps.Keys(o.vals))
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -70,7 +71,7 @@ func moduleAt(held map[string]delivery.Module, repository, dir string) (string,
|
||||
// moduleOfDir is the module of a changed directory of a pull request: the module its manifest in the message names, else
|
||||
// the one the modules bucket holds at that directory of the repository, else the directory's last element (the
|
||||
// catalogue's rule for nearly every manifest).
|
||||
func moduleOfDir(held map[string]delivery.Module, f delivery.Forge, dir string) string {
|
||||
func moduleOfDir(held map[string]delivery.Module, f forgemsg.Forge, dir string) string {
|
||||
if name := manifestModule(f.Manifests[dir]); name != "" {
|
||||
return name
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
@@ -297,6 +298,12 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st
|
||||
ix.mu.Lock()
|
||||
got, ok := ix.atHeads[at]
|
||||
ix.mu.Unlock()
|
||||
ix.mu.Lock()
|
||||
failed := ix.failedAt[repo] == head
|
||||
ix.mu.Unlock()
|
||||
if failed {
|
||||
continue // said once for this head; the candidate takes the published version until the head changes
|
||||
}
|
||||
if !ok {
|
||||
var ms []string
|
||||
ms = append(ms, modules[repo]...)
|
||||
@@ -307,10 +314,29 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st
|
||||
}
|
||||
var err error
|
||||
if got, err = ix.packagesAt(ctx, repo, head, ms, imports[repo]); err != nil {
|
||||
ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err)
|
||||
// A failure of the tree itself (the directory is not in the tree at that head) no retry changes: it is asked once per
|
||||
// head. Any other failure may be transient: it is asked again at the next pass, and said once per head.
|
||||
ix.mu.Lock()
|
||||
said := ix.saidAt[repo] == head
|
||||
if ix.saidAt == nil {
|
||||
ix.saidAt = map[string]string{}
|
||||
}
|
||||
ix.saidAt[repo] = head
|
||||
if errors.Is(err, ErrNotInTree) {
|
||||
if ix.failedAt == nil {
|
||||
ix.failedAt = map[string]string{}
|
||||
}
|
||||
ix.failedAt[repo] = head
|
||||
}
|
||||
ix.mu.Unlock()
|
||||
if !said {
|
||||
ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
ix.mu.Lock()
|
||||
delete(ix.failedAt, repo)
|
||||
delete(ix.saidAt, repo)
|
||||
if ix.atHeads == nil || len(ix.atHeads) >= maxHeadsKept {
|
||||
ix.atHeads = map[string]map[string]delivery.PublishedPackages{}
|
||||
}
|
||||
@@ -322,5 +348,9 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st
|
||||
return out
|
||||
}
|
||||
|
||||
// ErrNotInTree is a failure of the tree itself: a directory asked of the forge is not in the tree at that commit. No retry
|
||||
// changes it, so a head that fails so is asked once (candidatePublished).
|
||||
var ErrNotInTree = errors.New("the directory is not in the tree at that commit")
|
||||
|
||||
// maxHeadsKept bounds the member heads' packages kept between recomputes; past it they are read again.
|
||||
const maxHeadsKept = 64
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,651 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
|
||||
// takeInBed is an indexer holding the catalogue's avahi and the indexer's own module, both with a fingerprint, its
|
||||
// pending state and the take-in events it says; base is the catalogue merge it was built from.
|
||||
type takeInBed struct {
|
||||
ix *Indexer
|
||||
mods *memStore
|
||||
pending *memStore
|
||||
said []takein.TakeIn
|
||||
bridge map[string][]byte // what the forge bridge answers (bridge_merged_message), by repository@merge
|
||||
base forgemsg.Forge
|
||||
logs *strings.Builder
|
||||
}
|
||||
|
||||
func newTakeInBed(t *testing.T) *takeInBed {
|
||||
t.Helper()
|
||||
ix, mods, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")},
|
||||
mod{delivery.IndexerModule, renamed(t, "avahi", delivery.IndexerModule)})
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
|
||||
b := &takeInBed{ix: ix, mods: mods, pending: newMem(), bridge: map[string][]byte{}, base: f, logs: &strings.Builder{}}
|
||||
ix.Verdicts = newMem()
|
||||
b.pending.maxValue = buscompose.StateMaxValueBytes
|
||||
ix.Pending = b.pending
|
||||
ix.Message = func(_ context.Context, repository, merge string) ([]byte, error) {
|
||||
if data, ok := b.bridge[repository+"@"+merge]; ok {
|
||||
return data, nil
|
||||
}
|
||||
return nil, errors.New("the gate holds no merge of " + repository + " with that commit")
|
||||
}
|
||||
ix.SayTakeIn = func(_ context.Context, t takein.TakeIn) error { b.said = append(b.said, t); return nil }
|
||||
logf := ix.Logf
|
||||
ix.Logf = func(format string, args ...any) {
|
||||
b.logs.WriteString(format + "\n")
|
||||
if logf != nil {
|
||||
logf(format, args...)
|
||||
}
|
||||
}
|
||||
own := b.fp(t, delivery.IndexerModule)
|
||||
if own == "" || b.fp(t, "avahi") == "" {
|
||||
t.Fatalf("the bed's modules have no fingerprint")
|
||||
}
|
||||
ix.OwnBuild = func(context.Context) (string, error) { return own, nil }
|
||||
ix.OwnBuildState = newMem()
|
||||
ix.FixOwnBuild(context.Background()) // fixed once, at the start
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *takeInBed) fp(t *testing.T, module string) string {
|
||||
t.Helper()
|
||||
m, _ := moduleOf(t, b.mods, module)
|
||||
return m.FP2
|
||||
}
|
||||
|
||||
func (b *takeInBed) commit(t *testing.T, module string) string {
|
||||
t.Helper()
|
||||
m, _ := moduleOf(t, b.mods, module)
|
||||
return m.Commit
|
||||
}
|
||||
|
||||
// merge is a catalogue merge at commit c whose trees of avahi and the indexer are given ("" keeps the base's).
|
||||
func (b *takeInBed) merge(number int, c, avahi, indexer string) forgemsg.Forge {
|
||||
f := b.base
|
||||
f.Number, f.Merge, f.Head = number, c, c
|
||||
f.Trees = map[string]string{}
|
||||
for dir, tree := range b.base.Trees {
|
||||
f.Trees[dir] = tree
|
||||
}
|
||||
if avahi != "" {
|
||||
f.Trees["modules/avahi"] = avahi
|
||||
}
|
||||
if indexer != "" {
|
||||
f.Trees["modules/"+delivery.IndexerModule] = indexer
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func (b *takeInBed) arrive(t *testing.T, seq uint64, f forgemsg.Forge) {
|
||||
t.Helper()
|
||||
b.publish(t, f)
|
||||
if _, _, err := b.ix.Arrive(context.Background(), seq, f); err != nil {
|
||||
t.Fatalf("arrive %d: %v", seq, err)
|
||||
}
|
||||
}
|
||||
|
||||
// publish lets the bed's forge bridge answer the merged message f, as bridge_merged_message does.
|
||||
func (b *takeInBed) publish(t *testing.T, f forgemsg.Forge) {
|
||||
t.Helper()
|
||||
data, err := forgemsg.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.bridge[f.Repository+"@"+f.Merge] = data
|
||||
}
|
||||
|
||||
func (b *takeInBed) keys() []string {
|
||||
k, _ := b.pending.Keys(context.Background())
|
||||
slices.Sort(k)
|
||||
return k
|
||||
}
|
||||
|
||||
func (b *takeInBed) events() []string {
|
||||
var out []string
|
||||
for _, s := range b.said {
|
||||
out = append(out, s.Kind+" "+s.Key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sha(c byte) string { return strings.Repeat(string(c), 40) }
|
||||
|
||||
func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
|
||||
ctx := context.Background()
|
||||
t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer
|
||||
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
|
||||
b.ix.FixOwnBuild(ctx)
|
||||
b.said = nil
|
||||
_, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
|
||||
if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') {
|
||||
t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi"))
|
||||
}
|
||||
// The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the
|
||||
// build is unknown, from the own-build state.
|
||||
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") ||
|
||||
b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" {
|
||||
t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said)
|
||||
} else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil {
|
||||
t.Errorf("the unknown-build event is refused by its own contract: %v", err)
|
||||
}
|
||||
answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{})
|
||||
if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() {
|
||||
t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err)
|
||||
}
|
||||
// Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once,
|
||||
// as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever).
|
||||
b.ix.Pending = &memStore{unbound: process.ErrNotYet}
|
||||
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")); err != nil || b.commit(t, "avahi") != sha('b') {
|
||||
t.Errorf("no pending state: err %v, avahi at %s: want the merge taken in at once", err, b.commit(t, "avahi"))
|
||||
}
|
||||
})
|
||||
t.Run("T7 and T3 a merge alone, an indexer change, a merge behind it and a revert", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
own, avahi0 := b.fp(t, delivery.IndexerModule), b.fp(t, "avahi")
|
||||
// A merge with no change to the indexer and nothing pending is taken in alone, as before.
|
||||
b.arrive(t, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
|
||||
if avahi1 := b.fp(t, "avahi"); avahi1 == avahi0 || len(b.keys()) != 0 || len(b.said) != 0 {
|
||||
t.Fatalf("alone: avahi %s (was %s), pending %v, said %v: want taken in", avahi1, avahi0, b.keys(), b.events())
|
||||
}
|
||||
avahi1, verdicts := b.fp(t, "avahi"), len(b.ix.Verdicts.(*memStore).vals)
|
||||
// T4: a change to the indexer writes only its entry (with its graph key and verdict); the rest is pending.
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
named := b.fp(t, delivery.IndexerModule)
|
||||
if named == own || b.fp(t, "avahi") != avahi1 || b.commit(t, "avahi") != sha('b') {
|
||||
t.Fatalf("indexer change: indexer %s (own %s), avahi %s at %s: want only the indexer written", named, own, b.fp(t, "avahi"), b.commit(t, "avahi"))
|
||||
}
|
||||
if g := graphOf(t, b.ix, delivery.IndexerModule); g.FP != named {
|
||||
t.Errorf("graph of the indexer names %s, want %s", g.FP, named)
|
||||
}
|
||||
if n := len(b.ix.Verdicts.(*memStore).vals); n > verdicts+1 {
|
||||
t.Errorf("verdicts %d, want at most %d: only the new indexer manifest's", n, verdicts+1)
|
||||
}
|
||||
if want := []string{takein.PendingKey(2)}; !reflect.DeepEqual(b.keys(), want) || !slices.Equal(b.events(), []string{"pending " + takein.PendingKey(2)}) || b.said[0].WaitsFor != named {
|
||||
t.Fatalf("pending %v, said %+v: want m.2 pending for %s", b.keys(), b.said, named)
|
||||
}
|
||||
// T5: the merge delivered again writes no second key and says nothing again.
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
if len(b.keys()) != 1 || len(b.said) != 1 {
|
||||
t.Errorf("redelivered: pending %v, said %v", b.keys(), b.events())
|
||||
}
|
||||
// A later merge joins: the newest governs and still names the new indexer.
|
||||
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
if len(b.keys()) != 2 || b.commit(t, "avahi") != sha('b') || len(b.said) != 1 {
|
||||
t.Fatalf("behind: pending %v, avahi at %s, said %v: want both pending and nothing said again", b.keys(), b.commit(t, "avahi"), b.events())
|
||||
}
|
||||
// A revert brings the indexer back to the running build: everything is taken in, oldest first.
|
||||
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", b.base.Trees["modules/"+delivery.IndexerModule]))
|
||||
if len(b.keys()) != 0 || b.fp(t, delivery.IndexerModule) != own || b.commit(t, "avahi") != sha('e') {
|
||||
t.Fatalf("revert: pending %v, indexer %s, avahi at %s: want all taken in", b.keys(), b.fp(t, delivery.IndexerModule), b.commit(t, "avahi"))
|
||||
}
|
||||
if want := []string{"pending " + takein.PendingKey(2), "taken " + takein.PendingKey(2)}; !slices.Equal(b.events(), want) {
|
||||
t.Errorf("said %v, want %v", b.events(), want)
|
||||
}
|
||||
})
|
||||
t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) {
|
||||
// A successor's manifest invokes a verb of the node-engine's seat that the built-in list lacks and the revision's
|
||||
// definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending;
|
||||
// accepted with them, and the revision waits for the build it names.
|
||||
const verb, contracts = "manifest-reads", "modules/seat-contracts"
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
defined bool
|
||||
refused bool
|
||||
}{{"definitions the revision does not carry", false, true}, {"definitions the revision carries", true, false}} {
|
||||
b := newTakeInBed(t)
|
||||
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
|
||||
f.Manifests = map[string]string{}
|
||||
for dir, raw := range b.base.Manifests {
|
||||
f.Manifests[dir] = raw
|
||||
}
|
||||
dir := "modules/" + delivery.IndexerModule
|
||||
f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+delivery.IndexerModule+`"`,
|
||||
`"module": "`+delivery.IndexerModule+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1)
|
||||
if row.defined {
|
||||
f.Trees[contracts] = "tree-seat-contracts"
|
||||
f.Locks = map[string]map[string]string{}
|
||||
for d, l := range b.base.Locks {
|
||||
f.Locks[d] = l
|
||||
}
|
||||
f.Locks[contracts] = map[string]string{}
|
||||
f.Manifests[contracts] = `{"module":"seat-contracts","version":"1","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"generated-from":"."}]}}`
|
||||
b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) {
|
||||
if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) {
|
||||
t.Errorf("listed %v of %s at %s", dirs, repository, commit)
|
||||
}
|
||||
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil
|
||||
}
|
||||
b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
|
||||
switch p {
|
||||
case contracts + "/seats/node-engine/seat.json":
|
||||
return []byte(`{"format":1,"seat":"node-engine","scope":"node","includes":["reads"]}`), nil
|
||||
case contracts + "/parts/reads.json":
|
||||
return []byte(`{"format":1,"part":"reads","verbs":[{"name":"` + verb + `"}]}`), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
b.arrive(t, 2, f)
|
||||
waits := PendingWaits(readPending(t, b))
|
||||
refused := ""
|
||||
if len(b.said) > 0 {
|
||||
refused = b.said[0].Refused
|
||||
}
|
||||
if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) {
|
||||
t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused)
|
||||
}
|
||||
if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) {
|
||||
t.Errorf("%s: the seats are not put back after the dry run", row.name)
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
g := b.merge(30, sha('f'), "", "")
|
||||
g.Repository, g.Trees, g.Manifests, g.Locks, g.Group = "novox/mesh-sdk", map[string]string{}, nil, nil, "feat/x"
|
||||
g.Members = []delivery.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
|
||||
b.arrive(t, 5, g)
|
||||
named := b.fp(t, delivery.IndexerModule)
|
||||
// The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
|
||||
// meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again.
|
||||
mk := func(ask func(context.Context) (string, error)) *Indexer {
|
||||
n := &Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
|
||||
Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: b.ix.Message, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum}
|
||||
n.toolchains = b.ix.toolchains
|
||||
n.OwnBuild = ask
|
||||
n.FixOwnBuild(ctx)
|
||||
return n
|
||||
}
|
||||
// A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the
|
||||
// wait was said before; and indexer_pending names the known build and since when.
|
||||
oldBuild, _ := b.ix.OwnBuild(ctx)
|
||||
oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil })
|
||||
b.logs.Reset()
|
||||
if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") ||
|
||||
!strings.Contains(b.logs.String(), "at the start, this program runs build") {
|
||||
t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String())
|
||||
}
|
||||
if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() {
|
||||
t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err)
|
||||
}
|
||||
asked := 0
|
||||
next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") })
|
||||
if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") {
|
||||
t.Errorf("a hand ask while the own build is unknown: %+v", r)
|
||||
}
|
||||
if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
|
||||
t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys())
|
||||
}
|
||||
// The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start.
|
||||
next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil }
|
||||
if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
|
||||
t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys())
|
||||
}
|
||||
// The module started again fixes the build at its start and takes the revision in.
|
||||
next = mk(func(context.Context) (string, error) { return named, nil })
|
||||
if _, _, err := next.TakeInAtStart(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') {
|
||||
t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi"))
|
||||
}
|
||||
})
|
||||
t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.ix.Taken = newMem()
|
||||
// The commits in the order they were made: one is in another when it comes first.
|
||||
order := "a756bc"
|
||||
b.ix.Contains = func(_ context.Context, _, commit, in string) (bool, error) {
|
||||
return strings.IndexByte(order, commit[0]) <= strings.IndexByte(order, in[0]), nil
|
||||
}
|
||||
member := func(repo string, number int, c string) forgemsg.Forge {
|
||||
f := b.merge(number, c, "", "")
|
||||
f.Group = "feat/take-in"
|
||||
f.Members = []delivery.Head{{Repository: catalogueRepo, Number: 40, Head: sha('1')}, {Repository: "novox/mesh-sdk", Number: 41, Head: sha('2')}}
|
||||
if repo != catalogueRepo {
|
||||
f.Repository, f.Trees, f.Manifests, f.Locks = repo, map[string]string{}, nil, nil
|
||||
}
|
||||
f.Head = map[int]string{40: sha('1'), 41: sha('2')}[number]
|
||||
return f
|
||||
}
|
||||
b.arrive(t, 1, member(catalogueRepo, 40, sha('7')))
|
||||
if b.commit(t, "avahi") == sha('7') || !slices.Equal(b.events(), []string{"pending g.feat/take-in"}) {
|
||||
t.Fatalf("first member: avahi at %s, said %v: want held, the group said pending", b.commit(t, "avahi"), b.events())
|
||||
}
|
||||
// A merge in another repository is taken in alone; one in a member's repository waits behind the group.
|
||||
alone := b.merge(50, sha('5'), "", "")
|
||||
alone.Repository, alone.Trees, alone.Manifests, alone.Locks = "novox/hq", map[string]string{}, nil, nil
|
||||
b.arrive(t, 2, alone)
|
||||
b.arrive(t, 3, b.merge(51, sha('6'), "tree-avahi-9", ""))
|
||||
if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(1), takein.PendingKey(3)}) {
|
||||
t.Fatalf("pending %v: want the member and the merge behind it", b.keys())
|
||||
}
|
||||
end := forgemsg.ForgeGroupEnd{Group: "feat/take-in", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('7')}, {Repository: "novox/mesh-sdk", Number: 41, Merge: sha('b')}}}
|
||||
if _, _, err := b.ix.EndGroup(ctx, end); err == nil {
|
||||
t.Fatal("an end before its last member is taken in")
|
||||
}
|
||||
b.arrive(t, 4, member("novox/mesh-sdk", 41, sha('b')))
|
||||
if _, _, err := b.ix.EndGroup(ctx, end); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('6') {
|
||||
t.Fatalf("end: pending %v, avahi at %s: want the group and the merge behind it taken in", b.keys(), b.commit(t, "avahi"))
|
||||
}
|
||||
if want := []string{"pending g.feat/take-in", "pending g.feat/take-in", "taken g.feat/take-in"}; !slices.Equal(b.events(), want) {
|
||||
t.Errorf("said %v, want %v", b.events(), want)
|
||||
}
|
||||
// R1: the end published again after the take-in changes nothing and is not delivered again.
|
||||
if _, words, err := b.ix.EndGroup(ctx, end); err != nil || !strings.Contains(words, "taken in already") {
|
||||
t.Errorf("an end again: %q %v", words, err)
|
||||
}
|
||||
// T11 (the lead's row): a group whose revision changes the indexer: the group's wait is taken, then the revision pending.
|
||||
b.said = nil
|
||||
g := member(catalogueRepo, 40, sha('c'))
|
||||
g.Group, g.Trees = "feat/two", b.merge(0, "", "", "tree-indexer-7").Trees
|
||||
g.Members = g.Members[:1]
|
||||
b.arrive(t, 9, g)
|
||||
if _, _, err := b.ix.EndGroup(ctx, forgemsg.ForgeGroupEnd{Group: "feat/two", Members: []forgemsg.GroupMerge{{Repository: catalogueRepo, Number: 40, Merge: sha('c')}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := []string{"pending g.feat/two", "taken g.feat/two", "pending " + takein.PendingKey(9)}; !slices.Equal(b.events(), want) {
|
||||
t.Errorf("a group naming another indexer: said %v, want %v", b.events(), want)
|
||||
}
|
||||
})
|
||||
t.Run("T5 two take-ins of one pending revision: the one that loses the claim writes nothing", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
named := b.fp(t, delivery.IndexerModule)
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = named
|
||||
b.ix.ownMu.Unlock()
|
||||
// The other indexer claims the merge between this one's read and its claim.
|
||||
b.pending.beforeWrite = func() {
|
||||
v := b.pending.vals[takein.PendingKey(2)]
|
||||
var p Pending
|
||||
_ = json.Unmarshal(v, &p)
|
||||
p.Taking = "the other indexer"
|
||||
data, _ := json.Marshal(p)
|
||||
b.pending.put(takein.PendingKey(2), data)
|
||||
}
|
||||
if _, _, err := b.ix.TakeInAtStart(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.commit(t, "avahi") == sha('c') || len(b.keys()) != 1 || !strings.Contains(b.logs.String(), "is lost") {
|
||||
t.Errorf("avahi at %s, pending %v: want nothing taken in by the loser, and it said", b.commit(t, "avahi"), b.keys())
|
||||
}
|
||||
})
|
||||
t.Run("T5 a take-in whose store fails once leaves no claim behind, and is taken in after", func(t *testing.T) {
|
||||
claims := func(b *takeInBed) []string {
|
||||
var out []string
|
||||
for _, k := range b.keys() {
|
||||
var p Pending
|
||||
_ = json.Unmarshal(b.pending.vals[k], &p)
|
||||
if p.Taking != "" {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
once := func(match func(key string) bool) func(string) error {
|
||||
done := false
|
||||
return func(key string) error {
|
||||
if !done && match(key) {
|
||||
done = true
|
||||
return errors.New("the store failed once")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
fail func(b *takeInBed)
|
||||
}{
|
||||
// The second merge's claim fails: the first's, already made, is released.
|
||||
{"a claim lost", func(b *takeInBed) {
|
||||
b.pending.failWrite = once(func(k string) bool { return k == takein.PendingKey(3) })
|
||||
}},
|
||||
// The take-in itself fails on a module's write after both claims: both are released.
|
||||
{"a take-in that fails", func(b *takeInBed) { b.mods.failWrite = once(func(string) bool { return true }) }},
|
||||
} {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
named := b.fp(t, delivery.IndexerModule)
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = named
|
||||
b.ix.ownMu.Unlock()
|
||||
c.fail(b)
|
||||
_, _, _ = b.ix.TakeInAtStart(ctx)
|
||||
if got := claims(b); len(got) != 0 || len(b.keys()) != 2 {
|
||||
t.Fatalf("%s: pending %v, claimed %v: want both pending and no claim left", c.name, b.keys(), got)
|
||||
}
|
||||
// The next take-in, not at a start, is not refused by a claim left behind: it takes both in.
|
||||
b.arrive(t, 4, b.merge(23, sha('e'), "tree-avahi-5", "tree-indexer-2"))
|
||||
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('e') {
|
||||
t.Errorf("%s: after: pending %v, avahi at %s: want all taken in", c.name, b.keys(), b.commit(t, "avahi"))
|
||||
}
|
||||
}
|
||||
})
|
||||
t.Run("T7 a toolchain report evaluates pending again: the revision then names the running build and is taken in", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
entries, err := ReadPending(ctx, b.pending, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The build the revision names under the next toolchain, worked out on a dry indexer; this one runs it.
|
||||
x := b.ix.dry().ix
|
||||
for _, node := range []string{"novox", "ace"} {
|
||||
key, _ := delivery.ToolchainKey(node, "go")
|
||||
x.PutToolchain("mesh-build-go", key, toolchainValue(t, "go", "go1.27.2", "1"), false)
|
||||
}
|
||||
ready, _, _ := b.ix.loadNewest(ctx, WaitsOf(entries).Ready)
|
||||
next, _, err := x.computeOver(ctx, ready)
|
||||
if err != nil || next == "" || next == b.said[0].WaitsFor {
|
||||
t.Fatalf("the build under the next toolchain: %q %v (the revision names %s)", next, err, b.said[0].WaitsFor)
|
||||
}
|
||||
b.ix.ownMu.Lock()
|
||||
b.ix.ownFP = next
|
||||
b.ix.ownMu.Unlock()
|
||||
reportToolchain(t, b.ix, "mesh-build-go", "novox", "go", "go1.27.2")
|
||||
if len(b.keys()) != 1 {
|
||||
t.Fatalf("one node's report: pending %v, want the revision still held (the nodes disagree)", b.keys())
|
||||
}
|
||||
reportToolchain(t, b.ix, "mesh-build-go", "ace", "go", "go1.27.2")
|
||||
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('c') {
|
||||
t.Errorf("after the toolchain report: pending %v, avahi at %s: want the revision taken in", b.keys(), b.commit(t, "avahi"))
|
||||
}
|
||||
})
|
||||
t.Run("T12 T13 indexer_pending names the wait; a hand take-in and a drop say what they ended", func(t *testing.T) {
|
||||
b := newTakeInBed(t)
|
||||
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
|
||||
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
|
||||
var answer takein.PendingAnswer
|
||||
for _, tool := range PendingTools(b.pending, nil, nil, nil) {
|
||||
if tool.Name == PendingTool {
|
||||
out, err := tool.Run(map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answer = out.(takein.PendingAnswer)
|
||||
}
|
||||
}
|
||||
if len(answer.Waits) != 1 || answer.Waits[0].Key != takein.PendingKey(2) || !slices.Equal(answer.Waits[0].Holds, []string{"avahi", delivery.IndexerModule}) ||
|
||||
answer.Waits[0].Said != takein.TakeInPendingEvent || len(answer.Waits[0].Keys) != 2 {
|
||||
t.Fatalf("indexer_pending = %+v", answer)
|
||||
}
|
||||
b.said = nil
|
||||
if r := b.ix.DropPending(ctx, takein.PendingKey(2), "lead@shanks", "replaced by m.3"); !r.Stored {
|
||||
t.Fatalf("drop: %+v", r)
|
||||
}
|
||||
if len(b.said) < 2 || !b.said[0].Dropped || b.said[0].Key != takein.PendingKey(2) || b.said[0].By != "lead@shanks" || b.said[1].Key != takein.PendingKey(3) {
|
||||
t.Fatalf("drop said %+v: want m.2 taken (dropped, by) and m.3 pending", b.said)
|
||||
}
|
||||
b.said = nil
|
||||
if r := b.ix.TakeInPendingByHand(ctx, takein.PendingKey(3), "lead@shanks", "the named indexer can never run"); !r.Stored {
|
||||
t.Fatalf("hand take-in: %+v", r)
|
||||
}
|
||||
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || len(b.said) != 1 || b.said[0].Kind != takein.TakeInTaken || b.said[0].By != "lead@shanks" {
|
||||
t.Errorf("hand take-in: pending %v, avahi at %s, said %+v", b.keys(), b.commit(t, "avahi"), b.said)
|
||||
}
|
||||
})
|
||||
t.Run("1585 a merge over the value cap is held by reference, a new indexer with an empty memory takes it in through the forge bridge, and a covered pull request leaves its candidate", func(t *testing.T) {
|
||||
big := func(f forgemsg.Forge) forgemsg.Forge {
|
||||
f.Words = strings.Repeat("x", 300*1024)
|
||||
f.Branch = "feat/g"
|
||||
return f
|
||||
}
|
||||
b := newTakeInBed(t)
|
||||
avahi0 := b.fp(t, "avahi")
|
||||
// The forge bridge is not asked at all while the merges are in memory: a 300 KiB merge, over the bucket's cap whole,
|
||||
// is held by reference and a lone one taken in.
|
||||
bridgeDown := func(context.Context, string, string) ([]byte, error) {
|
||||
return nil, errors.New("the bridge does not answer")
|
||||
}
|
||||
answers := b.ix.Message
|
||||
b.ix.Message = bridgeDown
|
||||
b.arrive(t, 1, big(b.merge(20, sha('b'), "tree-avahi-2", "")))
|
||||
if b.fp(t, "avahi") == avahi0 || b.commit(t, "avahi") != sha('b') || len(b.keys()) != 0 || len(b.ix.held) != 0 {
|
||||
t.Fatalf("a big merge alone: avahi at %s, pending %v, in memory %d: want it taken in and forgotten", b.commit(t, "avahi"), b.keys(), len(b.ix.held))
|
||||
}
|
||||
// Two pull requests of one group are open; both then merge (the indexer changed: held for the new build).
|
||||
for n, head := range map[int]string{21: sha('1'), 22: sha('2')} {
|
||||
if err := b.ix.Apply(ctx, pull(catalogueRepo, n, "feat/g", head, "modules/avahi")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
key, _ := delivery.CandidatesKey("feat/g")
|
||||
if _, _, found, _ := b.ix.Store.Get(ctx, key); !found {
|
||||
t.Fatal("the group has no candidate")
|
||||
}
|
||||
b.arrive(t, 2, big(b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")))
|
||||
b.arrive(t, 3, big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
|
||||
raw, _, _, _ := b.pending.Get(ctx, takein.PendingKey(2))
|
||||
if len(b.keys()) != 2 || len(raw) > 4096 || b.commit(t, "avahi") != sha('b') {
|
||||
t.Fatalf("a big merge held: pending %v, value %d bytes, avahi at %s: want two small values", b.keys(), len(raw), b.commit(t, "avahi"))
|
||||
}
|
||||
named := b.fp(t, delivery.IndexerModule)
|
||||
// The new indexer starts with nothing in memory and the bridge down: the newest merge is said lost with the bridge's
|
||||
// words (event, state), the older one is covered and not lost, and nothing is taken in.
|
||||
next := &Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
|
||||
Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: bridgeDown, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf,
|
||||
GoSum: b.ix.GoSum, OwnBuild: func(context.Context) (string, error) { return named, nil }}
|
||||
next.toolchains = b.ix.toolchains
|
||||
next.FixOwnBuild(ctx)
|
||||
b.said = nil
|
||||
if _, _, err := next.TakeInAtStart(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waits := PendingWaits(mustReadPending(t, b))
|
||||
if len(b.said) != 1 || !slices.Equal(b.said[0].Lost, []string{takein.PendingKey(3)}) || len(waits) != 1 || !slices.Equal(waits[0].Lost, []string{takein.PendingKey(3)}) || b.commit(t, "avahi") != sha('b') {
|
||||
t.Fatalf("lost: said %+v, waits %+v, avahi at %s: want m.3 lost on the event and in indexer_pending, nothing taken in", b.said, waits, b.commit(t, "avahi"))
|
||||
}
|
||||
if v, _, _, _ := b.pending.Get(ctx, takein.PendingKey(3)); !strings.Contains(string(v), "the bridge does not answer") {
|
||||
t.Errorf("m.3 = %s: want the bridge's words", v)
|
||||
}
|
||||
if err := next.PendingAgain(ctx); err != nil || len(b.said) != 1 {
|
||||
t.Errorf("lost again: err %v, said %v: want it said once", err, b.events())
|
||||
}
|
||||
if r := next.TakeInPendingByHand(ctx, takein.PendingKey(2), "lead@shanks", "try"); r.Stored || b.commit(t, "avahi") != sha('b') {
|
||||
t.Errorf("hand take-in of a lost merge: %+v, avahi at %s: want nothing taken in", r, b.commit(t, "avahi"))
|
||||
}
|
||||
// The bridge answers: the newest merge is read through it, the older one is covered, and the revision is taken in.
|
||||
b.publish(t, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) // the answer is the message as published, small here
|
||||
next.Message = answers
|
||||
b.bridge[catalogueRepo+"@"+sha('d')], _ = forgemsg.EncodeForge(big(b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")))
|
||||
b.logs.Reset()
|
||||
if err := next.PendingAgain(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(b.keys()) != 0 || b.commit(t, "avahi") != sha('d') || !strings.Contains(b.logs.String(), "is covered by") {
|
||||
t.Fatalf("taken in: pending %v, avahi at %s, journal %q: want both merges taken in, the older covered", b.keys(), b.commit(t, "avahi"), b.logs.String())
|
||||
}
|
||||
if _, _, found, _ := b.ix.Store.Get(ctx, key); found {
|
||||
t.Errorf("the group's candidate stays after both its pull requests merged (the covered one is not taken out)")
|
||||
}
|
||||
// A hand take-in of a merge by its commit asks the bridge and runs the message through Arrive.
|
||||
b.ix.Message = answers
|
||||
b.publish(t, big(b.merge(30, sha('e'), "tree-avahi-6", b.base.Trees["modules/"+delivery.IndexerModule])))
|
||||
if r := b.ix.TakeInMerge(ctx, catalogueRepo, sha('e'), 9, "lead@shanks", "held by the cap"); !r.Stored || b.commit(t, "avahi") != sha('e') {
|
||||
t.Fatalf("merge take-in: %+v, avahi at %s: want it taken in", r, b.commit(t, "avahi"))
|
||||
}
|
||||
if r := b.ix.TakeInMerge(ctx, catalogueRepo, sha('9'), 10, "lead@shanks", "no such"); r.Stored || !strings.Contains(r.Answer, "gate holds no merge") {
|
||||
t.Errorf("merge take-in of a merge the bridge does not know: %+v", r)
|
||||
}
|
||||
// An old value that holds the message whole reads as before.
|
||||
old := b.merge(31, sha('f'), "tree-avahi-7", b.base.Trees["modules/"+delivery.IndexerModule])
|
||||
rawOld, _ := forgemsg.EncodeForge(old)
|
||||
data, _ := encodePending(Pending{Forge: rawOld, Since: b.ix.now(), LastWriter: b.fp(t, delivery.IndexerModule)})
|
||||
if err := b.pending.Create(ctx, takein.PendingKey(11), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := b.ix.PendingAgain(ctx); err != nil || b.commit(t, "avahi") != sha('f') {
|
||||
t.Errorf("an old value with the message whole: err %v, avahi at %s: want it taken in", err, b.commit(t, "avahi"))
|
||||
}
|
||||
})
|
||||
t.Run("the dry run names every field of the indexer", func(t *testing.T) {
|
||||
// dry (pending.go) builds an indexer field by field: a field added to Indexer is named there, or this fails.
|
||||
named := map[string]bool{}
|
||||
for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections",
|
||||
"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
|
||||
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile", "OpenTries", "TryRecovery", "OwnBuildState"} {
|
||||
named[f] = true
|
||||
}
|
||||
ty := reflect.TypeOf(Indexer{})
|
||||
for i := range ty.NumField() {
|
||||
if f := ty.Field(i); f.IsExported() && !named[f.Name] {
|
||||
t.Errorf("Indexer.%s is not named by dry", f.Name)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func mustReadPending(t *testing.T, b *takeInBed) []PendingEntry {
|
||||
t.Helper()
|
||||
entries, err := ReadPending(context.Background(), b.pending, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
// judgeSeatsHave says whether the catalogue's seat list holds the verb on the seat now.
|
||||
func judgeSeatsHave(seat, verb string) bool {
|
||||
s, _ := catalogue.SeatNamed(seat)
|
||||
return slices.ContainsFunc(s.Serves, func(v catalogue.Verb) bool { return v.Name == verb })
|
||||
}
|
||||
|
||||
// readPending is the bed's pending state as indexer_pending reads it.
|
||||
func readPending(t *testing.T, b *takeInBed) []PendingEntry {
|
||||
t.Helper()
|
||||
entries, err := ReadPending(context.Background(), b.pending, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return entries
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// PendingTools are the indexer's tools of what waits for a take-in (novox/hq ADR 0633 §5): indexer_pending reads the
|
||||
// pending state, and indexer_take_in_pending and indexer_drop_pending steer it by an ask in takeins, as indexer_take_in
|
||||
// does. pending is the pending state as the tools bundle reads it; ask writes an ask and waits for the indexer's answer.
|
||||
func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool {
|
||||
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
|
||||
hand := func(drop bool) func(map[string]any) (any, error) {
|
||||
return func(args map[string]any) (any, error) {
|
||||
key, value, err := NewPendingAsk(str(args, "key"), str(args, "why"), stdio.CallerOf(args), drop, now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}
|
||||
}
|
||||
merge := func(args map[string]any) (any, error) {
|
||||
seq, _ := args["sequence"].(float64)
|
||||
key, value, err := NewMergeAsk(str(args, "repository"), str(args, "merge"), uint64(seq), str(args, "why"), stdio.CallerOf(args), now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask(key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}
|
||||
return []stdio.Tool{
|
||||
{Name: PendingTool,
|
||||
Description: "What waits for a take-in (ADR 0633): merges the indexer holds in its pending state instead of taking them in. " +
|
||||
"A revision (what revision, its key its oldest merge's m.<sequence>) waits for the build of the indexer it names " +
|
||||
"(waits-for): the running indexer wrote only what building and moving that indexer needs, and the new indexer takes " +
|
||||
"it in at its start. A group (what group, key g.<group>) is a delivery group whose members are gathered until the " +
|
||||
"forge bridge says its end; a merge in one of its members' repositories waits behind it (behind). Each names its " +
|
||||
"merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " +
|
||||
"longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " +
|
||||
"every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " +
|
||||
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " +
|
||||
"runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
var unread []string
|
||||
entries, err := ReadPending(context.Background(), pending, func(format string, args ...any) {
|
||||
unread = append(unread, fmt.Sprintf(format, args...))
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}
|
||||
// Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why.
|
||||
if ownBuild != nil {
|
||||
if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found {
|
||||
var o OwnBuild
|
||||
if json.Unmarshal(v, &o) == nil {
|
||||
answer.OwnBuildSince = o.Since
|
||||
if o.Known {
|
||||
answer.OwnBuild = o.FP
|
||||
} else {
|
||||
answer.OwnBuildUnknown = o.Why
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return answer, nil
|
||||
}},
|
||||
{Name: TakeInPendingTool,
|
||||
Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " +
|
||||
"whatever build of the indexer it names, or a delivery group under its g. key with the members gathered so far. For " +
|
||||
"when the named indexer can never run, or a group's end never comes. A hand act: who and why are said in the " +
|
||||
"indexer's journal and on the take-in.taken event, which clears take-in-waits. indexer_pending names the keys.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
|
||||
"key": map[string]any{"type": "string", "description": "the wait's key, m.<sequence> or g.<group>, as indexer_pending names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
|
||||
Run: hand(false)},
|
||||
{Name: DropPendingTool,
|
||||
Description: "Drop one pending merge by hand, abandoned because a later merge replaced it (ADR 0633 §5): it is removed from " +
|
||||
"the pending state and nothing of it is taken in; what waited with it is worked out again. A hand act: who and why are " +
|
||||
"said in the journal and on the take-in.taken event (dropped).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"key", "why"}, "properties": map[string]any{
|
||||
"key": map[string]any{"type": "string", "description": "the pending merge's key, m.<sequence>, as indexer_pending names its merges' wait"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
|
||||
Run: hand(true)},
|
||||
{Name: TakeInMergeTool,
|
||||
Description: "Take one merge in by hand (ADR 0633 §5, issue 1585): the indexer asks the forge bridge for the merge's merged message " +
|
||||
"(bridge_merged_message) and runs it through the take-in as a delivered merged message, so it is held for the " +
|
||||
"indexer its revision names or taken in. For a merge the take-in did not take in (indexer_heard names its sequence " +
|
||||
"and why). A hand act: who and why are said in the indexer's journal. The bridge's own words answer when it cannot " +
|
||||
"build the message (the gate holds no such merge, the forge does not answer).",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "merge", "sequence", "why"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "the repository, owner/name"},
|
||||
"merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"},
|
||||
"sequence": map[string]any{"type": "integer", "description": "the merge's sequence on the forge bridge's stream, as indexer_heard names it"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is taken in by hand, in plain words"}}},
|
||||
Run: merge},
|
||||
}
|
||||
}
|
||||
|
||||
// TryCandidateTools are the tools of a group try's candidate (issue 1921, trycandidates.go): indexer_try_candidates reads the
|
||||
// candidates the indexer holds for tries (the try, its heads, each member's modules and whether each has its fingerprint),
|
||||
// and indexer_drop_try_candidate drops one by an ask in takeins. candidates is the candidates state as the tools bundle reads it.
|
||||
func TryCandidateTools(candidates, recovery Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: TryCandidatesTool,
|
||||
Description: "The candidates the indexer holds for group tries (a try of several repositories at exact commits, mesh-delivery-checker try): " +
|
||||
"per try its heads hash, each member's repository and commit, the modules the try touches and, per module, its manifest digest, its " +
|
||||
"fingerprint or why it has none yet (a merge preview not yet written). The checker's manifest validator and the node-engines' " +
|
||||
"compose-candidate read this candidate. Read-only; an empty answer means no whole try waits or runs.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) {
|
||||
ctx := context.Background()
|
||||
keys, err := candidates.Keys(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []delivery.Candidate
|
||||
for _, k := range keys {
|
||||
if !IsTryGroup(k) {
|
||||
continue
|
||||
}
|
||||
v, _, found, err := candidates.Get(ctx, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if c, err := delivery.DecodeCandidate(v); err == nil {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
// How the recovery of the open tries went: loud when it is not done (since when, why).
|
||||
said := map[string]any{"done": false, "why": "no recovery has been written yet"}
|
||||
if v, _, found, err := recovery.Get(ctx, TryRecoveryKey); err == nil && found {
|
||||
var r TryRecovery
|
||||
if json.Unmarshal(v, &r) == nil {
|
||||
said = map[string]any{"done": r.Done}
|
||||
if !r.Done {
|
||||
said["why"], said["since"] = r.Why, r.Since
|
||||
said["says"] = "recovery not done, since " + r.Since.Format(time.RFC3339) + ": " + r.Why
|
||||
}
|
||||
}
|
||||
}
|
||||
return map[string]any{"tries": out, "recovery": said}, nil
|
||||
}},
|
||||
{Name: RecoverTriesTool,
|
||||
Description: "Run the recovery of the open tries now: the indexer lists them through the checker's checker_tries, writes the candidate of each " +
|
||||
"whole try that waits or runs and deletes the candidate of a try no longer listed. It runs by itself at the indexer's start; when it failed " +
|
||||
"(indexer_try_candidates says recovery not done, since when and why) this runs it again. A hand act: who and why are said in the journal.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{
|
||||
"why": map[string]any{"type": "string", "description": "why it is run now, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
why, _ := args["why"].(string)
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return nil, fmt.Errorf("say why: a hand act is said in the journal")
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, RecoverTries: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask("tries.recover."+now().UTC().Format("20060102T150405.000000000"), data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
{Name: DropTryCandidateTool,
|
||||
Description: "Drop a group try's candidate by hand (the indexer deletes it itself when the try ends): for a candidate left for a try that " +
|
||||
"is gone, or to make the indexer write it again from the previews at the try's next change. A hand act: who and why are said in the journal.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"try", "why"}, "properties": map[string]any{
|
||||
"try": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
|
||||
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
id, _ := args["try"].(string)
|
||||
why, _ := args["why"].(string)
|
||||
id = strings.TrimSpace(id)
|
||||
if !IsTryGroup(id) || strings.TrimSpace(why) == "" {
|
||||
return nil, fmt.Errorf("name the try (try-<digits>) and say why")
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, TryCandidate: id, Drop: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a, err := ask("try-candidate.drop."+id, data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.Result, nil
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/sdk/grouppackages"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
@@ -72,7 +73,10 @@ func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, d
|
||||
if !ix.PutPreview(key, value, deleted) {
|
||||
return nil
|
||||
}
|
||||
return ix.RecomputeCandidates(ctx)
|
||||
if err := ix.RecomputeCandidates(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.WriteTryCandidates(ctx) // a try that waited for this preview gets its candidate
|
||||
}
|
||||
|
||||
// RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer
|
||||
@@ -315,7 +319,10 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
|
||||
out.None2 = none
|
||||
return out
|
||||
}
|
||||
out.FP2, out.Packages = fp, versions
|
||||
// The ask of the candidate's build names every member of the group at its head, also one the module names no package of
|
||||
// (a member that publishes none yet, or one the module reaches through mesh-sdk): the fingerprint stays the trunk's, so
|
||||
// the build is the one the merge keeps (grouppackages.Heads, issue 1921).
|
||||
out.FP2, out.Packages = fp, grouppackages.Heads(versions, members, pv.Repository)
|
||||
if len(inputs.Paths) > 0 {
|
||||
out.Paths = inputs.Paths // the builder's fetch for the candidate names them (ADR 0499 clause 4)
|
||||
}
|
||||
|
||||
@@ -3,10 +3,18 @@ package index
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"os"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/fingerprint"
|
||||
@@ -38,7 +46,7 @@ func avahiOnTools(t *testing.T) string {
|
||||
}
|
||||
|
||||
// avahiPull is the pull message of the catalogue's pull request 301 touching avahi, with its manifest.
|
||||
func avahiPull(t *testing.T, head string) delivery.Forge {
|
||||
func avahiPull(t *testing.T, head string) forgemsg.Forge {
|
||||
t.Helper()
|
||||
f := manifestPull(t, 301, head, avahiOnTools(t))
|
||||
f.Manifests = map[string]string{"modules/avahi": avahiOnTools(t)}
|
||||
@@ -101,6 +109,42 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
|
||||
t.Errorf("candidate base = %d, want the source revision 5 its preview was read onto", c.Base)
|
||||
}
|
||||
written(t, writtenCandidate{Source: sourceRev{Rev: 5, Heads: map[string]string{catalogueRepo: trunkNow}}, Candidate: c})
|
||||
// Issue 1921: a candidate's build is asked every member of the group at its head, also one that publishes no package yet
|
||||
// and one the module names no package of; the fingerprint stays the trunk's, so the build is the one the merge keeps.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
members map[string]string // the other pull requests of the group: repository, head
|
||||
want delivery.PackageVersions
|
||||
}{
|
||||
{"a group of one names nothing", nil, nil},
|
||||
{"a member that publishes no package yet", map[string]string{"novox/mesh-contracts": h2},
|
||||
delivery.PackageVersions{delivery.ForgeModulePrefix + "mesh-contracts": h2}},
|
||||
{"mesh-sdk by its go module", map[string]string{"novox/mesh-sdk": h2, "novox/mesh-contracts": h3},
|
||||
delivery.PackageVersions{delivery.ForgeModulePrefix + "mesh-sdk/go": h2, delivery.ForgeModulePrefix + "mesh-contracts": h3}},
|
||||
} {
|
||||
ix, mods, cands := previewBed(t, true)
|
||||
trunk, _ := moduleOf(t, mods, "avahi")
|
||||
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for repository, head := range tc.members {
|
||||
other := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgePull, Repository: repository, Branch: "feat/x", Number: 7, Head: head,
|
||||
Action: forgemsg.PullUpdated}
|
||||
if err := ix.Apply(context.Background(), other); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
|
||||
var got delivery.CandidateModule
|
||||
for _, p := range candidateOf(t, cands).Pulls {
|
||||
if p.Repository == catalogueRepo {
|
||||
got = p.Fps["avahi"]
|
||||
}
|
||||
}
|
||||
if got.FP2 != trunk.FP2 || !maps.Equal(got.Packages, tc.want) {
|
||||
t.Errorf("%s: candidate avahi = %+v, want the trunk's fingerprint %s and the packages %v", tc.name, got, trunk.FP2, tc.want)
|
||||
}
|
||||
}
|
||||
// Another tree gives another fingerprint.
|
||||
pv := avahiPreview(t, trunkNow, h1)
|
||||
m := pv.Modules["modules/avahi"]
|
||||
@@ -181,6 +225,256 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW
|
||||
if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NoneNoPreview {
|
||||
t.Errorf("a new head: %+v, want no preview", got)
|
||||
}
|
||||
// Issue 1921: a whole group try has no pull request, yet the indexer writes its candidate from the checker's try events and
|
||||
// the previews its fetch wrote (number 0, the heads hash of its members), keeps it apart from the pull requests' candidates,
|
||||
// and deletes it when the try ends. Its manifests are judged with the seat definitions its members carry, in one locked
|
||||
// pass, whether the contracts module's manifest changed or only its seat.json did (the wait is bounded: a lock held twice
|
||||
// would show as a failure here, not as a stuck run).
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
unchanged bool // the contracts module's module.json is the trunk's: only its definitions changed
|
||||
}{
|
||||
{name: "a changed manifest that carries definitions"},
|
||||
{name: "an unchanged manifest that carries definitions", unchanged: true},
|
||||
} {
|
||||
t.Run("a group try's candidate: "+row.name, func(t *testing.T) {
|
||||
ix, mods, cands := previewBed(t, true)
|
||||
ctx := context.Background()
|
||||
ix.Verdicts = newMem()
|
||||
var changed map[string]any
|
||||
if err := json.Unmarshal([]byte(catalogueManifest(t, "avahi")), &changed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
changed["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "go-seats", "kind": "package", "language": "go", "paths": []any{"go"}, "generated-from": "defs"}}}
|
||||
raw, _ := json.Marshal(changed)
|
||||
pv := avahiPreview(t, trunkNow, h1)
|
||||
pm := delivery.PreviewModule{Tree: "tree-avahi", Manifest: delivery.ManifestDigest(raw), Locks: map[string]string{"go.sum": avahiGoSum}}
|
||||
if row.unchanged {
|
||||
m, found := moduleOf(t, mods, "avahi")
|
||||
_ = found
|
||||
pm.Manifest = m.Manifest
|
||||
key, _ := delivery.ManifestsKey(m.Manifest)
|
||||
data, _ := delivery.EncodeManifest(delivery.Manifest{Module: "avahi", JSON: string(raw)})
|
||||
if _, _, _, err := ix.Manifests.Get(ctx, key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = ix.Manifests.(*memStore).Update(ctx, key, data, ix.Manifests.(*memStore).revs[key])
|
||||
}
|
||||
pv.Modules["modules/avahi"] = pm
|
||||
ix.Read = func(_ context.Context, _, _, dir string) (RepoRead, error) {
|
||||
return RepoRead{Manifests: map[string]string{dir: string(raw)}}, nil
|
||||
}
|
||||
listed := 0
|
||||
ix.Files = func(_ context.Context, _, commit string, dirs []string) (map[string]map[string]string, error) {
|
||||
listed++
|
||||
if commit != h1 || !slices.Equal(dirs, []string{"modules/avahi/defs/seats"}) {
|
||||
t.Errorf("listed %v at %s, want the member's head", dirs, commit)
|
||||
}
|
||||
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil
|
||||
}
|
||||
ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
|
||||
switch p {
|
||||
case "modules/avahi/defs/seats/node-engine/seat.json":
|
||||
return []byte(`{"format":1,"seat":"node-engine","scope":"node","includes":["reads"]}`), nil
|
||||
case "modules/avahi/defs/parts/reads.json":
|
||||
return []byte(`{"format":1,"part":"reads","verbs":[{"name":"try-verb"}]}`), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
started := `{"id":"try-7"}`
|
||||
_ = started
|
||||
members := []TryMember{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}
|
||||
held := func() []string { k, _ := cands.Keys(ctx); return k }
|
||||
if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 0 {
|
||||
t.Fatalf("a try before its preview: %v, candidates %v", err, held())
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); applyPreview(t, ix, pv) }()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
panic("writing the try's candidate hangs: the seat lock is held twice (a locked pass cannot be left by t.Fatal: the lock stays held)")
|
||||
}
|
||||
if listed == 0 {
|
||||
t.Error("the member's definitions were not read")
|
||||
}
|
||||
key, _ := delivery.CandidatesKey("try-7")
|
||||
rawc, _, found, _ := cands.Get(ctx, key)
|
||||
if !found {
|
||||
t.Fatalf("no candidate for the try after its preview: %v", held())
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(rawc)
|
||||
pull := delivery.CandidatePull{}
|
||||
if len(c.Pulls) == 1 {
|
||||
pull = c.Pulls[0]
|
||||
}
|
||||
if row.unchanged {
|
||||
if err != nil || c.Group != "try-7" || len(pull.Modules) != 0 {
|
||||
t.Fatalf("candidate = %+v, %v, want the try's group with no module touched", c, err)
|
||||
}
|
||||
} else if err != nil || c.Group != "try-7" || pull.Number != 0 || pull.Head.Head != h1 || pull.Fps["avahi"].Manifest == "" { // no fingerprint asserted: this mesh-sdk pin does not read generated-from yet
|
||||
t.Fatalf("candidate = %+v, %v, want one pull of number 0 with avahi's manifest", c, err)
|
||||
}
|
||||
if err := ix.Apply(ctx, forgemsg.Forge{Kind: forgemsg.ForgeOpen}); err != nil || len(held()) != 1 {
|
||||
t.Errorf("the open message left %v (%v), want the try's candidate alone", held(), err)
|
||||
}
|
||||
// The tools: one reads the candidates held for tries, one drops by an ask the indexer answers.
|
||||
asks := func(key string, value []byte) (TakeInAsk, error) {
|
||||
a, err := DecodeTakeIn(value)
|
||||
r := ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why)
|
||||
a.Result = &r
|
||||
return a, err
|
||||
}
|
||||
recovery := newMem()
|
||||
ix.TryRecovery = recovery
|
||||
tools := TryCandidateTools(cands, recovery, asks, time.Now)
|
||||
if out, err := tools[0].Run(map[string]any{}); err != nil || len(out.(map[string]any)["tries"].([]delivery.Candidate)) != 1 {
|
||||
t.Errorf("%s = %v, %v, want the try's candidate", tools[0].Name, out, err)
|
||||
}
|
||||
if _, err := tools[2].Run(map[string]any{"try": "feat/x", "why": "w"}); err == nil {
|
||||
t.Error("a branch was dropped as a try")
|
||||
}
|
||||
if out, err := tools[2].Run(map[string]any{"try": "try-7", "why": "by hand"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 {
|
||||
t.Errorf("drop = %v, %v, candidates %v", out, err, held())
|
||||
}
|
||||
// The start recovered from the checker's tool: an open whole try is written again, one the checker no longer
|
||||
// lists is deleted, and a try narrowed to a level is none.
|
||||
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
|
||||
return []tryevent.Open{{ID: "try-7", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}},
|
||||
{ID: "try-8", Whole: false, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h2, Trunk: trunkNow}}}}, nil
|
||||
}
|
||||
if err := ix.RecoverTries(ctx); err != nil || len(held()) != 1 {
|
||||
t.Fatalf("recovered: %v, candidates %v, want try-7's alone", err, held())
|
||||
}
|
||||
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil }
|
||||
if err := ix.RecoverTries(ctx); err != nil || len(held()) != 0 {
|
||||
t.Errorf("recovered with no open try: %v, candidates %v, want none", err, held())
|
||||
}
|
||||
// A recovery whose call fails is loud and deletes nothing: the candidate stays, indexer_try_candidates says recovery not
|
||||
// done since when and why, and the next try event runs the recovery again, which clears it; the tool runs it by hand.
|
||||
if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 {
|
||||
t.Fatalf("started: %v, candidates %v", err, held())
|
||||
}
|
||||
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
|
||||
return nil, errors.New("checker_tries does not answer")
|
||||
}
|
||||
if err := ix.RecoverTries(ctx); err == nil || len(held()) != 1 {
|
||||
t.Fatalf("a failed call: %v, candidates %v, want the error and the candidate kept", err, held())
|
||||
}
|
||||
if out, err := tools[0].Run(map[string]any{}); err != nil || !strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "recovery not done, since") ||
|
||||
!strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "checker_tries does not answer") {
|
||||
t.Errorf("%s = %v, %v, want recovery not done with since and why", tools[0].Name, out, err)
|
||||
}
|
||||
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil }
|
||||
if err := ix.StartTry(ctx, "try-11", members); err != nil || len(held()) != 1 || held()[0] != "try-11" {
|
||||
t.Fatalf("a try event after a failed recovery: %v, candidates %v, want the recovery run (try-7 not listed, gone) and try-11 written", err, held())
|
||||
}
|
||||
if out, _ := tools[0].Run(map[string]any{}); out.(map[string]any)["recovery"].(map[string]any)["done"] != true {
|
||||
t.Errorf("recovery still said not done after a success: %v", out)
|
||||
}
|
||||
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, errors.New("down") }
|
||||
recoverAsk := func(key string, value []byte) (TakeInAsk, error) {
|
||||
a, err := DecodeTakeIn(value)
|
||||
r := ix.RecoverTriesByHand(ctx, a.By, a.Why)
|
||||
a.Result = &r
|
||||
return a, err
|
||||
}
|
||||
byHand := TryCandidateTools(cands, recovery, recoverAsk, time.Now)
|
||||
if out, err := byHand[1].Run(map[string]any{"why": "the checker was replaced"}); err != nil || out.(*TakenIn).Stored || !strings.Contains(out.(*TakenIn).Answer, "recovery not done") {
|
||||
t.Errorf("%s with the checker down = %v, %v", byHand[1].Name, out, err)
|
||||
}
|
||||
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil }
|
||||
if out, err := byHand[1].Run(map[string]any{"why": "the checker is back"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 {
|
||||
t.Errorf("%s = %v, %v, candidates %v, want it recovered (try-11 not listed)", byHand[1].Name, out, err, held())
|
||||
}
|
||||
if err := ix.StartTry(ctx, "try-7", members); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// ADR 0638 §3, §4: the consumer exists before the call, so what happens between the call and the hearing is heard after
|
||||
// it. A try the call listed and whose start is then heard gets one candidate; a try started after the list gets one; a
|
||||
// try that ended during the call, listed as open, leaves none once its end is heard.
|
||||
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
|
||||
return []tryevent.Open{{ID: "try-7", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}},
|
||||
{ID: "try-9", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}}}, nil
|
||||
}
|
||||
if err := ix.RecoverTries(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, step := range []struct {
|
||||
what string
|
||||
do func() error
|
||||
want int
|
||||
}{
|
||||
{"the start heard after the call, of a try the call listed", func() error { return ix.StartTry(ctx, "try-7", members) }, 2},
|
||||
{"a start after the list", func() error { return ix.StartTry(ctx, "try-10", members) }, 3},
|
||||
{"the end heard of a try ended during the call", func() error { return ix.EndTry(ctx, "try-9") }, 2},
|
||||
} {
|
||||
if err := step.do(); err != nil || len(held()) != step.want {
|
||||
t.Fatalf("%s: %v, candidates %v, want %d", step.what, err, held(), step.want)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"try-10"} {
|
||||
if err := ix.EndTry(ctx, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 {
|
||||
t.Fatalf("started again: %v, candidates %v", err, held())
|
||||
}
|
||||
if err := ix.EndTry(ctx, "try-7"); err != nil || len(held()) != 0 {
|
||||
t.Errorf("an ended try: %v, candidates %v, want none", err, held())
|
||||
}
|
||||
// The hand recovery never overlaps a try event: a start heard while the call to the checker is out writes its candidate
|
||||
// before or after the sweep, never between the call and the sweep (which would delete it, the try not being listed).
|
||||
startDone := make(chan error, 1)
|
||||
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
|
||||
go func() { startDone <- ix.StartTry(ctx, "try-12", members) }()
|
||||
for i := 0; i < 2000 && len(held()) == 0; i++ {
|
||||
runtime.Gosched() // lets the start write if nothing holds it back
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
if r := ix.RecoverTriesByHand(ctx, "lead", "a test"); !r.Stored {
|
||||
t.Fatalf("the hand recovery = %+v", r)
|
||||
}
|
||||
if err := <-startDone; err != nil || len(held()) != 1 || held()[0] != "try-12" {
|
||||
t.Errorf("a start heard during the hand recovery: %v, candidates %v, want try-12 kept", err, held())
|
||||
}
|
||||
})
|
||||
}
|
||||
// Issue 1921: a head whose tree lacks the directory asked is asked once, however many passes read it, until the head changes;
|
||||
// any other failure (the forge does not answer) is asked again at every pass and said once per head.
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
err error
|
||||
askedAfterTwo int // the same head read twice
|
||||
askedAfterNew int // then another head
|
||||
linesAfterNew int
|
||||
}{
|
||||
{"the directory is not in the tree", fmt.Errorf("listing: %w", ErrNotInTree), 1, 2, 2},
|
||||
{"the forge does not answer", errors.New("the forge does not answer"), 2, 3, 2},
|
||||
} {
|
||||
t.Run("a failing head: "+row.name, func(t *testing.T) {
|
||||
ix, _, _ := previewBed(t, false)
|
||||
var logs strings.Builder
|
||||
ix.Logf = func(format string, args ...any) { logs.WriteString(fmt.Sprintf(format, args...) + "\n") }
|
||||
asked := 0
|
||||
ix.Files = func(context.Context, string, string, []string) (map[string]map[string]string, error) {
|
||||
asked++
|
||||
return nil, row.err
|
||||
}
|
||||
held := map[string]delivery.Module{"x": {Inputs2: &delivery.KeptInputs2{Packages: []string{"git.novox.be/novox/mesh-sdk/go"}}}}
|
||||
for pass, head := range []string{h1, h1, h2} {
|
||||
ix.candidatePublished(context.Background(), map[string]string{"novox/mesh-sdk": head}, held)
|
||||
if pass == 1 && (asked != row.askedAfterTwo || strings.Count(logs.String(), "its packages are not read") != 1) {
|
||||
t.Fatalf("after two passes over %s: asked %d (want %d), journal %q: want one line", head, asked, row.askedAfterTwo, logs.String())
|
||||
}
|
||||
}
|
||||
if asked != row.askedAfterNew || strings.Count(logs.String(), "its packages are not read") != row.linesAfterNew {
|
||||
t.Errorf("asked %d (want %d), journal %q (want %d lines)", asked, row.askedAfterNew, logs.String(), row.linesAfterNew)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -125,7 +126,7 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
|
||||
t.why = fmt.Sprintf("the manifest at %q of %s at %s names the module %q", dir, repository, commit, name)
|
||||
return t, nil
|
||||
}
|
||||
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
|
||||
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: repository, Merge: commit, Root: read.Root,
|
||||
Trees: map[string]string{dir: read.Trees[dir]},
|
||||
Manifests: map[string]string{dir: raw}}
|
||||
if l := read.Locks[dir]; len(l) > 0 {
|
||||
@@ -396,7 +397,7 @@ func (ix *Indexer) TakeInRepository(ctx context.Context, repository string) Take
|
||||
out.Answer = "the modules state cannot be read: " + err.Error()
|
||||
return out
|
||||
}
|
||||
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: out.Repository, Merge: head}
|
||||
f := forgemsg.Forge{V: delivery.Version, Kind: forgemsg.ForgeMerged, Repository: out.Repository, Merge: head}
|
||||
if _, err := ix.ApplyMessage(ctx, f); err != nil {
|
||||
out.Answer = fmt.Sprintf("the merge of %s at %.12s could not be taken in: %v", repository, head, err)
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package index
|
||||
|
||||
// The seat definitions a revision carries (novox/hq ADR 0633 §1, ADR 0635 §3, ADR 0636 §4): when a take-in works out which
|
||||
// indexer a revision names, the manifests it writes are judged against the seats of that revision, never against the
|
||||
// list this program was built with, since the indexer the revision names may serve or invoke what the revision's
|
||||
// definitions add (a verb of a seat, an event). They are read from the merges' trees: each contracts module a merge's
|
||||
// manifests name (a package artifact with `generated-from`) holds `seats/<seat>/seat.json` and the `parts/` they
|
||||
// include, read through the forge's listing and get-file tools. A delivery group's merges are one revision, so a
|
||||
// group's members' definitions are read together.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"mesh-delivery-indexer/judge"
|
||||
)
|
||||
|
||||
// definitionsDir is the directory of a contracts module's definitions that the module.json at dir names, from the first
|
||||
// package artifact naming `generated-from`: ok is false for a module that is no contracts module. A manifest that does
|
||||
// not decode is none here: the judge says why it is refused.
|
||||
func definitionsDir(dir, manifest string) (string, bool) {
|
||||
var m struct {
|
||||
Build struct {
|
||||
Artifacts []struct {
|
||||
GeneratedFrom string `json:"generated-from"`
|
||||
} `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
if json.Unmarshal([]byte(manifest), &m) != nil {
|
||||
return "", false
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.GeneratedFrom == "" {
|
||||
continue
|
||||
}
|
||||
base := dir
|
||||
if base == "." || base == "/" {
|
||||
base = ""
|
||||
}
|
||||
return path.Clean(path.Join(base, a.GeneratedFrom)), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// revisionSeats are the seats of the definitions the ready merges carry, read at their merge commits, the newest merge
|
||||
// of a repository governing; none where the indexer has no forge reader. A read the forge does not answer is an error
|
||||
// (the take-in is tried again); a definition that does not parse is said and left out, as the indexer that keeps it
|
||||
// would say it unparsed.
|
||||
func (ix *Indexer) revisionSeats(ctx context.Context, ready []PendingEntry) ([]catalogue.Seat, error) {
|
||||
if ix.ReadFile == nil || ix.Files == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var defs []judge.Definition
|
||||
for _, e := range ready {
|
||||
if e.Covered || e.Forge.Merge == "" {
|
||||
continue
|
||||
}
|
||||
for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) {
|
||||
defsDir, ok := definitionsDir(dir, e.Forge.Manifests[dir])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
got, err := ix.readDefinitions(ctx, e.Forge.Repository, e.Forge.Merge, defsDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defs = append(defs, got...)
|
||||
}
|
||||
}
|
||||
seats, err := judge.Seats(defs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return seats, nil
|
||||
}
|
||||
|
||||
// readDefinitions reads the definitions under a contracts module's definitions directory at a commit, resolved with their
|
||||
// parts.
|
||||
func (ix *Indexer) readDefinitions(ctx context.Context, repository, commit, defsDir string) ([]judge.Definition, error) {
|
||||
file := func(p string) ([]byte, error) {
|
||||
if defsDir != "." && defsDir != "" {
|
||||
p = defsDir + "/" + p
|
||||
}
|
||||
raw, err := ix.ReadFile(ctx, repository, commit, p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s of %s at %.12s: %w", p, repository, commit, err)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
dir := path.Join(defsDir, "seats")
|
||||
if defsDir == "." || defsDir == "" {
|
||||
dir = "seats"
|
||||
}
|
||||
listed, err := ix.Files(ctx, repository, commit, []string{dir})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("listing %s of %s at %.12s: %w", dir, repository, commit, err)
|
||||
}
|
||||
parts := map[string]judge.Part{}
|
||||
var out []judge.Definition
|
||||
for _, p := range slices.Sorted(maps.Keys(listed[dir])) {
|
||||
seat, ok := strings.CutSuffix(p, "/seat.json")
|
||||
if !ok || seat == "" || strings.Contains(seat, "/") {
|
||||
continue
|
||||
}
|
||||
raw, err := file("seats/" + seat + "/seat.json")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d, err := judge.ParseDefinition(raw)
|
||||
if err == nil && d.Seat != seat {
|
||||
err = fmt.Errorf("it names the seat %q, not %q", d.Seat, seat)
|
||||
}
|
||||
for _, name := range d.Includes {
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if _, have := parts[name]; have {
|
||||
continue
|
||||
}
|
||||
praw, perr := file("parts/" + name + ".json")
|
||||
if perr != nil {
|
||||
return nil, perr
|
||||
}
|
||||
var part judge.Part
|
||||
if praw == nil {
|
||||
perr = errors.New("the commit holds no such part")
|
||||
} else {
|
||||
part, perr = judge.ParsePart(praw)
|
||||
}
|
||||
if perr != nil {
|
||||
err = fmt.Errorf("its part %s: %w", name, perr)
|
||||
break
|
||||
}
|
||||
parts[name] = part
|
||||
}
|
||||
if err == nil {
|
||||
d, err = d.Resolve(parts)
|
||||
}
|
||||
if err != nil {
|
||||
ix.logf("take-in: the seat definition seats/%s of %s at %.12s does not parse, so it is left out: %v", seat, repository, commit, err)
|
||||
continue
|
||||
}
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -51,6 +51,17 @@ func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.
|
||||
}
|
||||
|
||||
func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T) { // 1344/AC1 (the source first)
|
||||
// A source revision older than the one held is left (merged here from a test of its own, for the test budget).
|
||||
t.Run("an older source revision is left", func(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
_ = ix.ApplySource(ctx, 9, sourceOf("novox/mesh-delivery", merge))
|
||||
_ = ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge))
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
|
||||
t.Errorf("module a Src = %d, want 9", m.Src)
|
||||
}
|
||||
})
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge)); err != nil {
|
||||
@@ -93,14 +104,3 @@ func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344
|
||||
t.Errorf("module a = %+v, want the new merge with Src 0", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASourceRevisionOlderThanTheOneHeldIsLeft(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
_ = ix.ApplySource(ctx, 9, sourceOf("novox/mesh-delivery", merge))
|
||||
_ = ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge))
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
|
||||
t.Errorf("module a Src = %d, want 9", m.Src)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -33,11 +34,24 @@ type TakeInAsk struct {
|
||||
// Judge asks the manifests to be judged again now (indexer_judge_again) instead of the module to be taken in.
|
||||
Judge bool `json:"judge,omitempty"`
|
||||
// Digest names, with Judge, the one manifest digest of the module to judge (indexer_judge with a digest).
|
||||
Digest string `json:"digest,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Result *TakenIn `json:"result,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
Digest string `json:"digest,omitempty"`
|
||||
// Pending asks the wait under this take-in key to be taken in by hand (indexer_take_in_pending), or with Drop the
|
||||
// one pending merge under it to be dropped (indexer_drop_pending), instead of a module (pending.go, ADR 0633 §5).
|
||||
Pending string `json:"pending,omitempty"`
|
||||
// Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message
|
||||
// is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence.
|
||||
Merge string `json:"merge,omitempty"`
|
||||
MergeRepository string `json:"merge-repository,omitempty"`
|
||||
Sequence uint64 `json:"sequence,omitempty"`
|
||||
// TryCandidate asks the candidate of this try's group to be dropped (indexer_drop_try_candidate), trycandidates.go.
|
||||
TryCandidate string `json:"try-candidate,omitempty"`
|
||||
// RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries).
|
||||
RecoverTries bool `json:"recover-tries,omitempty"`
|
||||
Drop bool `json:"drop,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Result *TakenIn `json:"result,omitempty"`
|
||||
Answered time.Time `json:"answered,omitzero"`
|
||||
}
|
||||
|
||||
// Done says the ask is answered.
|
||||
@@ -52,8 +66,8 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
|
||||
if a.V != 1 {
|
||||
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
|
||||
}
|
||||
if a.Module == "" && a.Repository == "" {
|
||||
return a, errors.New("a takeins value naming no module or repository")
|
||||
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries {
|
||||
return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try")
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
@@ -82,3 +96,49 @@ func NewTakeInRepository(repository, why, by string, at time.Time) (string, []by
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Repository: repository, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return "repository." + key, data, err
|
||||
}
|
||||
|
||||
// TakeInMergeTool takes one merge in by hand, its merged message asked of the forge bridge (ADR 0633 §5, issue 1585).
|
||||
const TakeInMergeTool = "indexer_take_in_merge"
|
||||
|
||||
var sha40 = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
||||
|
||||
// NewMergeAsk is the value indexer_take_in_merge writes: its key in takeins, `merge.<commit>`, and the value. sequence is
|
||||
// the merge's sequence on the forge bridge's stream, as indexer_heard names it: the pending merge is keyed by it, which
|
||||
// keeps the merge order.
|
||||
func NewMergeAsk(repository, merge string, sequence uint64, why, by string, at time.Time) (string, []byte, error) {
|
||||
repository, merge = strings.TrimSpace(repository), strings.TrimSpace(merge)
|
||||
switch {
|
||||
case !strings.Contains(repository, "/"):
|
||||
return "", nil, errors.New("name the repository as owner/name (repository)")
|
||||
case !sha40.MatchString(merge):
|
||||
return "", nil, errors.New("name the merge commit, 40 hex digits (merge)")
|
||||
case sequence == 0:
|
||||
return "", nil, errors.New("name the merge's sequence on the forge bridge's stream (sequence), as indexer_heard names it")
|
||||
case strings.TrimSpace(why) == "":
|
||||
return "", nil, errors.New("say why (why): a hand act is said in the journal")
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Merge: merge, MergeRepository: repository, Sequence: sequence, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return "merge." + merge, data, err
|
||||
}
|
||||
|
||||
// NewPendingAsk is the value indexer_take_in_pending (drop false) or indexer_drop_pending (drop true) writes: its key
|
||||
// in takeins, `pending.<take-in|drop>.<key>`, and the value. key is a wait's key as indexer_pending names it, for a
|
||||
// drop one pending merge's.
|
||||
func NewPendingAsk(key, why, by string, drop bool, at time.Time) (string, []byte, error) {
|
||||
key = strings.TrimSpace(key)
|
||||
if !strings.HasPrefix(key, "m.") && !(strings.HasPrefix(key, "g.") && !drop) {
|
||||
if drop {
|
||||
return "", nil, errors.New("name the pending merge to drop by its key, m.<sequence>, as indexer_pending names it")
|
||||
}
|
||||
return "", nil, errors.New("name the wait to take in by its key, m.<sequence> or g.<group>, as indexer_pending names it")
|
||||
}
|
||||
if strings.TrimSpace(why) == "" {
|
||||
return "", nil, errors.New("say why (why): a hand act on what waits is said in the journal and on its take-in event")
|
||||
}
|
||||
act := "take-in"
|
||||
if drop {
|
||||
act = "drop"
|
||||
}
|
||||
data, err := json.Marshal(TakeInAsk{V: 1, Pending: key, Drop: drop, Why: strings.TrimSpace(why), By: by, At: at.UTC()})
|
||||
return "pending." + act + "." + key, data, err
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
)
|
||||
@@ -19,13 +20,13 @@ import (
|
||||
// say keeps the key as it is and says so.
|
||||
|
||||
// writeTaken writes the repository's taken key for the merge f carries, unless it names that merge or a later one.
|
||||
func (ix *Indexer) writeTaken(ctx context.Context, f delivery.Forge) error {
|
||||
func (ix *Indexer) writeTaken(ctx context.Context, f forgemsg.Forge) error {
|
||||
return ix.writeTakenKnown(ctx, f, "")
|
||||
}
|
||||
|
||||
// writeTakenKnown is writeTaken where the merge is known not to be older than notOlder (lateMerge asked it, issue 1126): a
|
||||
// key naming that merge is written over without asking the forge again.
|
||||
func (ix *Indexer) writeTakenKnown(ctx context.Context, f delivery.Forge, notOlder string) error {
|
||||
func (ix *Indexer) writeTakenKnown(ctx context.Context, f forgemsg.Forge, notOlder string) error {
|
||||
if ix.Taken == nil || f.Merge == "" {
|
||||
return nil
|
||||
}
|
||||
@@ -101,7 +102,7 @@ func (ix *Indexer) FillTaken(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
for _, repository := range slices.Sorted(maps.Keys(pending)) {
|
||||
err := ix.writeTaken(ctx, delivery.Forge{Repository: repository, Merge: pending[repository]})
|
||||
err := ix.writeTaken(ctx, forgemsg.Forge{Repository: repository, Merge: pending[repository]})
|
||||
if err == nil {
|
||||
ix.mu.Lock()
|
||||
if ix.untaken[repository] == pending[repository] {
|
||||
|
||||
@@ -78,12 +78,22 @@ func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool)
|
||||
}
|
||||
|
||||
// ApplyToolchain takes one change of a build module's toolchains in (PutToolchain) and, when the language's toolchain
|
||||
// changed, writes the modules again that it moves (Recompute). The same version and recipe seen again writes nothing.
|
||||
// changed, writes the modules again that it moves and evaluates pending again (ToolchainReported). The same version and
|
||||
// recipe seen again writes nothing.
|
||||
func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value []byte, deleted bool) error {
|
||||
if !ix.PutToolchain(module, key, value, deleted) {
|
||||
return nil
|
||||
}
|
||||
return ix.Recompute(ctx)
|
||||
return ix.ToolchainReported(ctx)
|
||||
}
|
||||
|
||||
// ToolchainReported is what a changed toolchain moves: the modules written again (Recompute), then pending evaluated
|
||||
// again (PendingAgain), since the indexer build a pending revision names is a fingerprint the toolchain is part of.
|
||||
func (ix *Indexer) ToolchainReported(ctx context.Context) error {
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return ix.PendingAgain(ctx)
|
||||
}
|
||||
|
||||
// agreed is the toolchain of each language every reporting node agrees on, or why there is none.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user