Compare commits
55
Commits
fa4be27c1e
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b04ee76473 | ||
|
|
2000c841bf | ||
|
|
46702d1af3 | ||
|
|
1239199b35 | ||
|
|
dcf62b0413 | ||
|
|
d68d7ef3d9 | ||
|
|
6c2b79acf1 | ||
|
|
bfc64ac65e | ||
|
|
e6ce4be4b4 | ||
|
|
ebac167cd6 | ||
|
|
27e4c3891f | ||
|
|
f555a05cfb | ||
|
|
3bd2a065f9 | ||
|
|
0b0cfb6e37 | ||
|
|
46c9f33d64 | ||
|
|
71bb5a5fb3 | ||
|
|
3298ad58d3 | ||
|
|
74d11f95a2 | ||
|
|
1e6d00b4c7 | ||
|
|
87e550681f | ||
|
|
c3804b012f | ||
|
|
07a28b03c4 | ||
|
|
26de6bc625 | ||
|
|
a5da6079de | ||
|
|
127ac64cce | ||
|
|
74fef7dd5a | ||
|
|
340bf64388 | ||
|
|
c9f742d5f2 | ||
|
|
f69a561a16 | ||
|
|
27bfd83fa0 | ||
|
|
a75d9cc851 | ||
|
|
ba7ce55887 | ||
|
|
3c1e7ad464 | ||
|
|
41225bf459 | ||
|
|
543d91d626 | ||
|
|
d650a41ba6 | ||
|
|
2520cc4a88 | ||
|
|
c93954e53c | ||
|
|
bf59b5a27b | ||
|
|
14e9d56a22 | ||
|
|
34f585106f | ||
|
|
05c180cf87 | ||
|
|
409c41d4ae | ||
|
|
5ba1da5b82 | ||
|
|
8ee56dbb5c | ||
|
|
be5c1b371e | ||
|
|
eeb5561a23 | ||
|
|
a0883a174b | ||
|
|
7686cfc55d | ||
|
|
507fac41fe | ||
|
|
9c15112d93 | ||
|
|
3c655c5ab4 | ||
|
|
8ffcf02a9c | ||
|
|
6a23627426 | ||
|
|
4b8e71370c |
@@ -0,0 +1,159 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// bucketOpener is the part of the bus the watches use.
|
||||
type bucketOpener interface {
|
||||
KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error)
|
||||
}
|
||||
|
||||
// kvStore is the builds bucket as the builder's store: create and compare-and-set.
|
||||
type kvStore struct{ kv jetstream.KeyValue }
|
||||
|
||||
func conflict(err error) error {
|
||||
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
|
||||
return build.ErrConflict
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s kvStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
e, err := s.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func (s kvStore) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
rev, err := s.kv.Create(ctx, key, value)
|
||||
return rev, conflict(err)
|
||||
}
|
||||
|
||||
func (s kvStore) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
rev, err := s.kv.Update(ctx, key, value, revision)
|
||||
return rev, conflict(err)
|
||||
}
|
||||
|
||||
func (s kvStore) Keys(ctx context.Context) ([]string, error) {
|
||||
keys, err := s.kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return keys, err
|
||||
}
|
||||
|
||||
func opOf(e jetstream.KeyValueEntry) bool { return e.Operation() == jetstream.KeyValuePut }
|
||||
|
||||
// settle reads the entries a watcher holds now, up to its mark of the last values, and returns the watcher for the
|
||||
// changes that follow.
|
||||
func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
|
||||
w, err := kv.WatchAll(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.Stop()
|
||||
return nil, ctx.Err()
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return nil, errors.New("the watch ended")
|
||||
}
|
||||
if e == nil {
|
||||
return w, nil
|
||||
}
|
||||
each(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// follow hands each change of a watcher to each until ctx ends; a watch that ends is an error, and the process manager
|
||||
// starts the program again.
|
||||
func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) error {
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return errors.New("a watch ended")
|
||||
}
|
||||
if e != nil {
|
||||
each(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stepsWaiter waits for a build module's step by watching its key in the module's steps bucket (ADR 0469 clause 2): the
|
||||
// watch hands over the value the key holds now, so a step that ended before the watch began is read at once, and every
|
||||
// change after. No timer: the caller bounds the wait with its context.
|
||||
type stepsWaiter struct{ js bucketOpener }
|
||||
|
||||
func (w stepsWaiter) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
|
||||
kv, err := w.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return delivery.Step{}, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
watch, err := kv.Watch(ctx, key)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
defer watch.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return delivery.Step{}, ctx.Err()
|
||||
case e, ok := <-watch.Updates():
|
||||
if !ok {
|
||||
return delivery.Step{}, errors.New("the watch of the step ended")
|
||||
}
|
||||
if e == nil || !opOf(e) {
|
||||
continue
|
||||
}
|
||||
step, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
return delivery.Step{}, fmt.Errorf("the step %s is not a step: %w", key, err)
|
||||
}
|
||||
if step.State != delivery.StepRunning {
|
||||
return step, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// manifestReader reads manifests.<digest> of the indexer and parses the manifest it holds.
|
||||
type manifestReader struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.Manifest, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
e, err := m.kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, fmt.Errorf("manifests.%s: %w", key, err)
|
||||
}
|
||||
kept, err := delivery.DecodeManifest(e.Value())
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.ParseManifest([]byte(kept.JSON))
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
// controller's user and the builder's own principals <node>.<module> (two instances, for the test of one build).
|
||||
const busConf = `
|
||||
listen: 127.0.0.1:-1
|
||||
jetstream { store_dir: %q }
|
||||
accounts {
|
||||
MESH {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "ctl", password: "pw" }
|
||||
{ user: "n1.mesh-delivery-builder", password: "pw" }
|
||||
{ user: "n2.mesh-delivery-builder", password: "pw" }
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
const (
|
||||
commit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20"
|
||||
fp = "src2:3333333333333333333333333333333333333333333333333333333333333333"
|
||||
)
|
||||
|
||||
// fakeBuildModules serve the tools of mesh-build-go and mesh-build-node as the real ones do (ADR 0469 clause 2): a call on
|
||||
// mesh.mod.<module>.call.<tool>.<node>.<caller> answers at once with the step and its key, and the step's end is written to
|
||||
// the key <node>.<id> of the module's steps bucket, some milliseconds later.
|
||||
type fakeBuildModules struct {
|
||||
js jetstream.JetStream
|
||||
ctx context.Context
|
||||
n atomic.Int64
|
||||
failBuild atomic.Bool
|
||||
mu sync.Mutex
|
||||
calls map[string]int // by `<module>.<tool>`
|
||||
}
|
||||
|
||||
func (f *fakeBuildModules) count(tool string) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.calls[tool]
|
||||
}
|
||||
|
||||
// digestOf is what every node builds an artifact to.
|
||||
func digestOf(artifact string) string {
|
||||
sum := sha256.Sum256([]byte(artifact))
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
|
||||
steps, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sub, err := nc.Subscribe("mesh.mod."+module+".call.>", func(m *nats.Msg) {
|
||||
tokens := strings.Split(m.Subject, ".") // mesh mod <module> call <tool> <node> <caller>
|
||||
tool, node := tokens[4], tokens[5]
|
||||
f.mu.Lock()
|
||||
f.calls[module+"."+tool]++
|
||||
f.mu.Unlock()
|
||||
var args map[string]any
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
id := fmt.Sprintf("s%d", f.n.Add(1))
|
||||
key, _ := delivery.StepKey(node, id)
|
||||
put := func(s delivery.Step) {
|
||||
data, err := delivery.EncodeStep(s)
|
||||
if err == nil {
|
||||
_, err = steps.Put(f.ctx, key, data)
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("writing step %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: time.Now().UTC()}
|
||||
put(step)
|
||||
answer, _ := json.Marshal(map[string]any{"result": map[string]any{"step": id, "key": key, "state": "running"}, "error": "", "node": node})
|
||||
_ = m.Respond(answer)
|
||||
go func() {
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
step.Ended = time.Now().UTC()
|
||||
switch {
|
||||
case strings.HasSuffix(tool, "_fetch"):
|
||||
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"}`)
|
||||
case f.failBuild.Load() && node == "n1":
|
||||
step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"])
|
||||
default:
|
||||
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+digestOf(fmt.Sprint(args["artifact"]))+`"}`)
|
||||
}
|
||||
put(step)
|
||||
}()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
}
|
||||
|
||||
type bed struct {
|
||||
ctx context.Context
|
||||
js jetstream.JetStream
|
||||
modules jetstream.KeyValue
|
||||
builds jetstream.KeyValue
|
||||
tools map[string]jetstream.KeyValue
|
||||
presence jetstream.KeyValue
|
||||
have jetstream.KeyValue
|
||||
fake *fakeBuildModules
|
||||
srvURL string
|
||||
dir string
|
||||
}
|
||||
|
||||
// startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the
|
||||
// build modules' streams and the builder's durable consumers on them, two nodes' toolchains, the indexer's manifests, and
|
||||
// the fake build modules. It runs no builder.
|
||||
func startBed(t *testing.T) *bed {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "b")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
conf := filepath.Join(dir, "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
srv, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(ctl.Close)
|
||||
js, err := jetstream.New(ctl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
b := &bed{ctx: ctx, js: js, tools: map[string]jetstream.KeyValue{}, srvURL: srv.ClientURL(), dir: dir}
|
||||
for module, stream := range toolchainStreams {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.stream." + module + ".event.toolchain.reported"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: node + "_mesh-delivery-builder", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket(module), History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.tools[module] = kv
|
||||
}
|
||||
mk := func(name string, history int) jetstream.KeyValue {
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
|
||||
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
||||
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
||||
presence := mk(delivery.PresenceBucket, delivery.PresenceHistory)
|
||||
b.presence, b.have = presence, mk(delivery.HaveBucket, delivery.HaveHistory)
|
||||
for _, node := range []string{"n1", "n2"} {
|
||||
b.setOnline(t, node, true)
|
||||
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
|
||||
b.setHealth(t, node, module, delivery.HealthOK)
|
||||
}
|
||||
}
|
||||
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
|
||||
b.fake.serve(t, ctl, "mesh-build-go")
|
||||
b.fake.serve(t, ctl, "mesh-build-node")
|
||||
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
|
||||
b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "1")
|
||||
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
|
||||
b.report(t, "mesh-build-node", "n2", "typescript", "tsc5", "1")
|
||||
return b
|
||||
}
|
||||
|
||||
// setOnline writes the watchman's presence of a node.
|
||||
func (b *bed) setOnline(t *testing.T, node string, online bool) {
|
||||
t.Helper()
|
||||
data, _ := delivery.EncodePresence(delivery.Presence{Online: online})
|
||||
if _, err := b.presence.Put(b.ctx, node, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// setHealth writes the node-engine's have of a build module on a node, running with this health.
|
||||
func (b *bed) setHealth(t *testing.T, node, module string, health delivery.Health) {
|
||||
t.Helper()
|
||||
data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: health})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.HaveKey(node, module)
|
||||
if _, err := b.have.Put(b.ctx, key, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does.
|
||||
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
|
||||
t.Helper()
|
||||
data, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now().UTC()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := delivery.ToolchainKey(node, language)
|
||||
if _, err := b.tools[module].Put(b.ctx, key, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.js.Publish(b.ctx, "mesh.stream."+module+".event.toolchain.reported", []byte(`{}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// module writes modules.<name> as the indexer does for a module with a new-form fingerprint, and its manifest by digest.
|
||||
func (b *bed) module(t *testing.T, name, fixture, path string) string {
|
||||
return b.moduleWith(t, name, fixture, path, map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}})
|
||||
}
|
||||
|
||||
// moduleWith is module with the toolchains its fingerprint was computed with.
|
||||
func (b *bed) moduleWith(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed) string {
|
||||
return b.moduleCommit(t, name, fixture, path, toolchains, commit)
|
||||
}
|
||||
|
||||
func (b *bed) moduleCommit(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed, at string) string {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join("testdata", fixture))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := delivery.ManifestDigest(raw)
|
||||
manifests, err := b.js.KeyValue(b.ctx, delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mv, err := delivery.EncodeManifest(delivery.Manifest{Module: name, JSON: string(raw)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := manifests.Put(b.ctx, digest, mv); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: at, FP: "src1:" + strings.Repeat("f", 64), FP2: fp,
|
||||
Manifest: digest, Path: path, Toolchains: toolchains})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.modules.Put(b.ctx, name, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
}
|
||||
|
||||
// archive writes modules.<name> as the real indexer does for a module of an archive only: its fingerprint names no
|
||||
// toolchain (mesh-delivery-indexer internal/index inputs.go languagesOf gives an archive no language), so Toolchains is empty.
|
||||
func (b *bed) archive(t *testing.T, name, path, at string) string {
|
||||
t.Helper()
|
||||
return b.moduleCommit(t, name, "catalogue-a67f0196-ca-trust.module.json", path, nil, at)
|
||||
}
|
||||
|
||||
// start runs the program as the builder of a node.
|
||||
func (b *bed) start(t *testing.T, node string, env map[string]string) {
|
||||
t.Helper()
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
for k, v := range env {
|
||||
e[k] = v
|
||||
}
|
||||
runCtx, stop := context.WithCancel(b.ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return e[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
}
|
||||
|
||||
// await reads the build key until pred holds of it, within 20 s.
|
||||
func (b *bed) await(t *testing.T, module string, pred func(delivery.BuilderBuild) bool) delivery.BuilderBuild {
|
||||
t.Helper()
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var last delivery.BuilderBuild
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
e, err := b.builds.Get(b.ctx, key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if last, err = delivery.DecodeBuilderBuild(e.Value()); err == nil && pred(last) {
|
||||
return last
|
||||
}
|
||||
}
|
||||
t.Fatalf("builds.%s = %+v within 20 s", key, last)
|
||||
return last
|
||||
}
|
||||
|
||||
func isState(state string) func(delivery.BuilderBuild) bool {
|
||||
return func(v delivery.BuilderBuild) bool { return v.State == state }
|
||||
}
|
||||
|
||||
// TestBuilderMakesADoneKeyWithTheDigests: modules.netcheck (a Go and a TypeScript bundle) and its manifest are in the
|
||||
// indexer's buckets, and two nodes report each build module's toolchain; the program writes builds.netcheck.<fp> done,
|
||||
// the digests of both bundles, both build modules' toolchains, after a fetch and a build on each of the two nodes.
|
||||
func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7
|
||||
b := startBed(t)
|
||||
digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "netcheck", isState(delivery.BuildDone))
|
||||
if v.By != "n1.mesh-delivery-builder" || v.Manifest != digest || len(v.Artifacts) != 2 {
|
||||
t.Fatalf("build = %+v", v)
|
||||
}
|
||||
for _, a := range v.Artifacts {
|
||||
if a.Digest != digestOf(a.Name) || a.Kind != "bundle" || a.Reference != "" {
|
||||
t.Errorf("artifact = %+v, want the digest every node built and no reference", a)
|
||||
}
|
||||
}
|
||||
if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" {
|
||||
t.Errorf("toolchains = %+v", v.Toolchains)
|
||||
}
|
||||
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 2, "mesh-build-go.build_go_build": 2, "mesh-build-node.build_node_fetch": 2, "mesh-build-node.build_node_build": 2} {
|
||||
if got := b.fake.count(tool); got != want {
|
||||
t.Errorf("%s called %d times, want %d (once on each of two nodes)", tool, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey: two builders on two nodes start together on one module key of an
|
||||
// archive only (no toolchain named, as the indexer writes it); the second takes over the key the first holds building
|
||||
// (ADR 0478 clause 6), and the key ends done, written once.
|
||||
func TestTwoBuilderInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
b.start(t, "n2", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files") {
|
||||
t.Fatalf("build = %+v, want the archive's digest", v)
|
||||
}
|
||||
if n := b.fake.count("mesh-build-go.build_go_build"); n < 2 || n%2 != 0 { // each instance that built, on two nodes
|
||||
t.Errorf("build_go_build was called %d times, want two for each instance that built", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt: the indexer gives an archive no language, so modules.ca-trust has no
|
||||
// toolchains; the builder builds it on the build module's nodes all the same (review round 4).
|
||||
func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (") {
|
||||
t.Errorf("build = %+v, want done on two nodes naming no toolchain", v)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAKeyBuiltOnOneNodeIsBuiltAgainWhenASecondNodeComesOnline: n2 is offline, so the key is done on n1 alone and marked;
|
||||
// n2 coming online makes the builder build it again on both nodes, as a new revision (ADR 0478 clause 4).
|
||||
func TestAKeyBuiltOnOneNodeIsBuiltAgainWhenASecondNodeComesOnline(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.setOnline(t, "n2", false)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
one := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if !strings.HasPrefix(one.Step, "one node (") {
|
||||
t.Fatalf("build = %+v, want it marked as built on one node", one)
|
||||
}
|
||||
key := mustKey(t, "ca-trust")
|
||||
e, err := b.builds.Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.setOnline(t, "n2", true)
|
||||
two := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool {
|
||||
return v.State == delivery.BuildDone && !strings.HasPrefix(v.Step, "one node (")
|
||||
})
|
||||
if e2, _ := b.builds.Get(b.ctx, key); e2.Revision() <= e.Revision() || !strings.Contains(two.Step, "n1 and n2") {
|
||||
t.Errorf("build = %+v at revision %d (was %d), want a new revision built on n1 and n2", two, e2.Revision(), e.Revision())
|
||||
}
|
||||
}
|
||||
|
||||
// TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed: the node-engine's have of mesh-build-go on n2 says failed, so only n1 builds.
|
||||
func TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.setHealth(t, "n2", "mesh-build-go", delivery.HealthFailed)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
if !strings.Contains(v.Step, "on one node, n1") {
|
||||
t.Errorf("build = %+v, want it built on n1 alone", v)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails on one node, so the
|
||||
// key is failed naming it; the indexer writing modules.<m> again with the same fingerprint wakes the builder, which builds
|
||||
// the key again as a new revision of the same key, now done.
|
||||
func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7
|
||||
b := startBed(t)
|
||||
b.fake.failBuild.Store(true)
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ca-trust", isState(delivery.BuildFailed))
|
||||
if !strings.Contains(v.Error, "n1") || !strings.Contains(v.Error, "boom") {
|
||||
t.Errorf("build = %+v, want failed naming the node and the step's reason", v)
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey("ca-trust", fp)
|
||||
failed, err := b.builds.Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.fake.failBuild.Store(false)
|
||||
// The indexer writes the module again (here at a later commit), with the same fingerprint.
|
||||
b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef")
|
||||
b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
done, err := b.builds.Get(b.ctx, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if done.Revision() <= failed.Revision() {
|
||||
t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision())
|
||||
}
|
||||
}
|
||||
|
||||
// TestAToolchainChangeDoesNotBuildTheOldFingerprint: the fingerprint names go1.27.1, the nodes then agree on go1.27.2 (a
|
||||
// report of both nodes), and the builder writes no key for the old fingerprint although it was woken by the report; a
|
||||
// module value naming go1.27.2 (the new fingerprint, computed by the indexer) is built.
|
||||
func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/AC4
|
||||
b := startBed(t)
|
||||
b.report(t, "mesh-build-go", "n1", "go", "go1.27.2", "1")
|
||||
b.report(t, "mesh-build-go", "n2", "go", "go1.27.2", "1")
|
||||
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") // names go1.27.1
|
||||
b.start(t, "n1", nil)
|
||||
// Another module, whose fingerprint names the new toolchain, is built; by then the old one has been looked at.
|
||||
b.moduleCommit(t, "netcheck2", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck",
|
||||
map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}, commit)
|
||||
b.await(t, "netcheck2", isState(delivery.BuildDone))
|
||||
if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil {
|
||||
t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2")
|
||||
}
|
||||
if n := b.fake.count("mesh-build-go.build_go_build"); n != 2 {
|
||||
t.Errorf("build_go_build was called %d times, want 2 (the new fingerprint on two nodes only)", n)
|
||||
}
|
||||
}
|
||||
|
||||
// TestARootModuleIsBuilt: a module whose path is "." is given to the build tools as it is.
|
||||
func TestARootModuleIsBuilt(t *testing.T) { // 1615/AC4
|
||||
b := startBed(t)
|
||||
b.archive(t, "ca-trust", delivery.RootPath, commit)
|
||||
b.start(t, "n1", nil)
|
||||
b.await(t, "ca-trust", isState(delivery.BuildDone))
|
||||
}
|
||||
|
||||
func mustKey(t *testing.T, module string) string {
|
||||
k, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// moduleAt is module with the go toolchain version the fingerprint names and the commit it was computed at.
|
||||
func (b *bed) moduleAt(t *testing.T, name, fixture, path, goVersion, at string) {
|
||||
t.Helper()
|
||||
b.moduleCommit(t, name, fixture, path, map[string]delivery.ToolchainUsed{"go": {Version: goVersion, Recipe: "1"}}, at)
|
||||
}
|
||||
|
||||
// TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart: the builds bucket holds a key left building by another node for
|
||||
// a module the indexer no longer has; the program, at start, fails it naming why (issue 1660, ADR 0478 clause 6).
|
||||
func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
|
||||
b := startBed(t)
|
||||
data, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "n2.mesh-delivery-builder", Manifest: strings.Repeat("a", 64)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := b.builds.Put(b.ctx, mustKey(t, "ghost"), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.start(t, "n1", nil)
|
||||
v := b.await(t, "ghost", isState(delivery.BuildFailed))
|
||||
if !strings.Contains(v.Error, "no longer has this fingerprint") || !strings.Contains(v.Error, "n2.mesh-delivery-builder") {
|
||||
t.Errorf("build = %+v, want it closed naming who left it", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
// Command mesh-delivery-builder is delivery's builder (novox/hq ADR 0453 clause 4, ADR 0478, issue 1607): it builds every
|
||||
// new-form fingerprint the indexer writes, dark, into its own build keys (builds.<module>.<fingerprint>), through the build
|
||||
// modules' fetch and build tools. Nothing reads its keys to deliver until release R4e. It serves no request subject and
|
||||
// holds no timer: every reaction is a watch or a bus event (ADR 0437 §8).
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultStepBound is how long the builder waits for one step of a build module (a fetch downloads the Go modules, a
|
||||
// build compiles).
|
||||
defaultStepBound = 30 * time.Minute
|
||||
// nodesPerFingerprint is how many build nodes build each fingerprint until release R4e (ADR 0478 clause 4).
|
||||
nodesPerFingerprint = 2
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "mesh-delivery-builder:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
|
||||
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
|
||||
type credential struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func readCredential(path string) (credential, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return credential{}, err
|
||||
}
|
||||
var c credential
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
|
||||
}
|
||||
if c.URL == "" || c.User == "" {
|
||||
return credential{}, errors.New("the broker credential names no bus address or user")
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinned accepts exactly the certificate with this SHA-256 and no other.
|
||||
func pinned(want string) *tls.Config {
|
||||
return &tls.Config{
|
||||
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
|
||||
MinVersion: tls.VersionTLS12,
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("the bus presented no certificate")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
|
||||
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||
logger := log.New(logOut, "mesh-delivery-builder: ", log.LstdFlags)
|
||||
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||
if path == "" {
|
||||
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
|
||||
}
|
||||
cred, err := readCredential(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts := []nats.Option{
|
||||
nats.Name(cred.Node + "." + cred.Module),
|
||||
nats.UserInfo(cred.User, cred.Password),
|
||||
nats.CustomInboxPrefix("_INBOX." + cred.User),
|
||||
nats.MaxReconnects(-1),
|
||||
nats.RetryOnFailedConnect(true),
|
||||
}
|
||||
if strings.TrimSpace(cred.Fingerprint) != "" {
|
||||
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
|
||||
}
|
||||
conn, err := nats.Connect(cred.URL, opts...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bucket := func(name string) (jetstream.KeyValue, error) {
|
||||
kv, err := js.KeyValue(ctx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
return kv, nil
|
||||
}
|
||||
builds, err := bucket(delivery.BuilderBuildsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
modules, err := bucket(delivery.ModulesBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := bucket(delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the
|
||||
// buckets are read and written on the connection above.
|
||||
account, err := process.ReadAccount(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
caller, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer caller.Close()
|
||||
|
||||
bound := defaultStepBound
|
||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
||||
if bound, err = time.ParseDuration(v); err != nil {
|
||||
return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err)
|
||||
}
|
||||
}
|
||||
capacity, err := capacityFrom(strings.TrimSpace(getenv("MESH_BUILDER_SETTINGS")), getenv("MESH_BUILDER_CAPACITY"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the build capacity setting: %w", err)
|
||||
}
|
||||
tools := &build.ToolchainSet{}
|
||||
presence := &build.PresenceSet{}
|
||||
have := &build.HaveSet{}
|
||||
b := &build.Builder{
|
||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||
Runner: build.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: bound},
|
||||
Toolchains: tools,
|
||||
Presence: presence,
|
||||
Health: have,
|
||||
Places: &build.Places{Capacity: capacity},
|
||||
Manifests: manifestReader{manifests},
|
||||
Nodes: nodesPerFingerprint,
|
||||
Logf: logger.Printf,
|
||||
}
|
||||
// The toolchains first, so a build that starts from the first modules value finds the nodes.
|
||||
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
|
||||
if err := watch.start(); err != nil {
|
||||
return err
|
||||
}
|
||||
// A build module says each time it wrote its toolchains key: its stream, and the builder's durable consumer on it, are
|
||||
// made by the controller. Where the stream is not there yet the builder says so and is started again with the build
|
||||
// module.
|
||||
for _, module := range []string{build.GoBuildModule, build.NodeBuildModule} {
|
||||
stream := toolchainStreams[module]
|
||||
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the builder starts again: %v", cred.Node, cred.Module, stream, module, err)
|
||||
continue
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cc.Stop()
|
||||
}
|
||||
// The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing
|
||||
// builds; the builder says so and is started again with the watchman.
|
||||
presenceErrs := make(chan error, 1)
|
||||
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no node counts as online until the builder starts again", delivery.PresenceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
|
||||
default:
|
||||
pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
presenceErrs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
b.NodesChanged(ctx) // a node came online or went: a build that waited for one may go on
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
// The node-engine's have says whether a node's build module runs and is healthy: a node whose build module is not is not
|
||||
// chosen (ADR 0478 clause 3). Where its bucket is not there yet no build module counts as healthy, so nothing builds.
|
||||
haveErrs := make(chan error, 1)
|
||||
switch kv, err := js.KeyValue(ctx, delivery.HaveBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no build module counts as healthy until the builder starts again", delivery.HaveBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.HaveBucket, err)
|
||||
default:
|
||||
hw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { have.Put(e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() {
|
||||
haveErrs <- follow(ctx, hw, func(e jetstream.KeyValueEntry) {
|
||||
if have.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||
b.NodesChanged(ctx) // a build module became healthy or stopped being: a build that waited may go on
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
b.Module(ctx, e.Key(), e.Value())
|
||||
} else {
|
||||
b.Forget(e.Key())
|
||||
}
|
||||
}
|
||||
w, err := settle(ctx, modules, each)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A key left building for a fingerprint its module no longer has is closed (issue 1660); the modules are all known now.
|
||||
if err := b.Sweep(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Printf("started on %s", cred.Node)
|
||||
errs := make(chan error, 1)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
b.Wait()
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
case err := <-watch.errs:
|
||||
return err
|
||||
case err := <-presenceErrs:
|
||||
return err
|
||||
case err := <-haveErrs:
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `<node>=<steps at
|
||||
// once>` pairs separated by commas, laid into the settings file the controller composes (issue 1661); empty is one step at a
|
||||
// time on every node. With no settings file named, fallback is read instead, for a run outside the module's launch.
|
||||
func capacityFrom(path, fallback string) (map[string]int, error) {
|
||||
if path == "" {
|
||||
return build.ParseCapacity(fallback)
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the settings file: %w", err)
|
||||
}
|
||||
var s struct {
|
||||
Capacity string `json:"capacity"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return nil, fmt.Errorf("the settings file %s is not JSON: %w", path, err)
|
||||
}
|
||||
return build.ParseCapacity(s.Capacity)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
func settingsFile(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "settings.json")
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// 1661: the build capacity is the module's setting `capacity`, laid into the settings file: empty is one step at a time everywhere.
|
||||
func TestCapacityIsReadFromTheSettingsFile(t *testing.T) {
|
||||
for body, want := range map[string]map[string]int{
|
||||
`{"capacity": "shanks=3,ace=2"}`: {"shanks": 3, "ace": 2},
|
||||
`{"capacity": ""}`: {},
|
||||
`{}`: {},
|
||||
} {
|
||||
got, err := capacityFrom(settingsFile(t, body), "")
|
||||
if err != nil || len(got) != len(want) || got["shanks"] != want["shanks"] || got["ace"] != want["ace"] {
|
||||
t.Errorf("%s: capacity %v, %v, want %v", body, got, err, want)
|
||||
}
|
||||
}
|
||||
p := &build.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))}
|
||||
if p.CapacityOf("shanks") != 1 || p.CapacityOf("ace") != 1 {
|
||||
t.Errorf("an empty setting gives %d and %d, want 1 everywhere", p.CapacityOf("shanks"), p.CapacityOf("ace"))
|
||||
}
|
||||
}
|
||||
|
||||
func mustCapacity(t *testing.T, path string) map[string]int {
|
||||
t.Helper()
|
||||
c, err := capacityFrom(path, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestACapacitySettingThatIsNotOneIsRefusedNamingIt(t *testing.T) {
|
||||
if _, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=-1"}`), ""); err == nil {
|
||||
t.Error("shanks=-1 accepted")
|
||||
}
|
||||
if c, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=3,ace=2,g14=0,novox=0"}`), ""); err != nil || c["g14"] != 0 || len(c) != 4 {
|
||||
t.Errorf("the starting value: %v, %v", c, err)
|
||||
}
|
||||
if _, err := capacityFrom(settingsFile(t, `not json`), ""); err == nil {
|
||||
t.Error("a file that is not JSON accepted")
|
||||
}
|
||||
if _, err := capacityFrom(filepath.Join(t.TempDir(), "none"), ""); err == nil {
|
||||
t.Error("a settings file that is not there accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// With no settings file named (a run outside the module's launch, as the tests do), the capacity is the environment's.
|
||||
func TestWithNoSettingsFileTheCapacityIsTheEnvironments(t *testing.T) {
|
||||
got, err := capacityFrom("", "ace=2")
|
||||
if err != nil || got["ace"] != 2 {
|
||||
t.Errorf("capacity %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"module": "ca-trust",
|
||||
"version": "1",
|
||||
"slug": "catrust",
|
||||
"capabilities": [
|
||||
"service-manager"
|
||||
],
|
||||
"requires": [
|
||||
"internal-acme-ca"
|
||||
],
|
||||
"seats": [
|
||||
{
|
||||
"name": "the-mesh-trust-anchor",
|
||||
"scope": "node"
|
||||
}
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "the-mesh-trust-anchor",
|
||||
"scope": "node"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "anchor",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/anchor",
|
||||
"mode": "0755",
|
||||
"template": "files/anchor"
|
||||
},
|
||||
{
|
||||
"id": "unit",
|
||||
"type": "file",
|
||||
"path": "/etc/systemd/system/mesh-ca-trust.service",
|
||||
"mode": "0644",
|
||||
"template": "files/mesh-ca-trust.service"
|
||||
},
|
||||
{
|
||||
"id": "trust",
|
||||
"type": "service",
|
||||
"unit": "mesh-ca-trust.service",
|
||||
"state": "running",
|
||||
"boot": "enabled",
|
||||
"restart-on": [
|
||||
"anchor",
|
||||
"unit"
|
||||
]
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"module": "netcheck",
|
||||
"version": "1",
|
||||
"emits": [
|
||||
"tunnel.endpoint-changed",
|
||||
"tunnel.handshake-stale",
|
||||
"tunnel.handshake-fresh"
|
||||
],
|
||||
"tools": [
|
||||
"netcheck_tunnel",
|
||||
"netcheck_tcp",
|
||||
"netcheck_dns",
|
||||
"netcheck_http",
|
||||
"netcheck_listening"
|
||||
],
|
||||
"replaces": {
|
||||
"netcheck_listening": [
|
||||
"ss -lunt",
|
||||
"ss -ltn",
|
||||
"netstat -lnt"
|
||||
],
|
||||
"netcheck_dns": [
|
||||
"dig",
|
||||
"nslookup"
|
||||
],
|
||||
"netcheck_tunnel": [
|
||||
"wg show",
|
||||
"wg show mesh0 dump"
|
||||
]
|
||||
},
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools-go",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/netcheck",
|
||||
"binary": "netcheck",
|
||||
"loads": [
|
||||
"netcheck"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "tools-typescript",
|
||||
"kind": "bundle",
|
||||
"language": "typescript",
|
||||
"entrypoints": [
|
||||
"tools/index.js"
|
||||
],
|
||||
"loads": [
|
||||
"tools/index.js"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-builder/internal/build"
|
||||
)
|
||||
|
||||
// toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it
|
||||
// (declared by the build modules, heard by this module through consumes).
|
||||
var toolchainStreams = map[string]string{build.GoBuildModule: "TOOLCHAINS_GO", build.NodeBuildModule: "TOOLCHAINS_NODE"}
|
||||
|
||||
// toolchainWatch keeps the build modules' toolchains in a build.ToolchainSet and tells the builder when one changed. A
|
||||
// build module's bucket is made when it is assigned, after the builder started, and the builder holds no timer (ADR 0437
|
||||
// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
|
||||
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched.
|
||||
type toolchainWatch struct {
|
||||
ctx context.Context
|
||||
js bucketOpener
|
||||
set *build.ToolchainSet
|
||||
changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again
|
||||
logger *log.Logger
|
||||
errs chan error
|
||||
mu sync.Mutex
|
||||
watched map[string]bool
|
||||
}
|
||||
|
||||
func newToolchainWatch(ctx context.Context, js bucketOpener, set *build.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
|
||||
return &toolchainWatch{ctx: ctx, js: js, set: set, changed: changed, logger: logger, errs: make(chan error, len(build.LanguageOf)), watched: map[string]bool{}}
|
||||
}
|
||||
|
||||
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
|
||||
func (w *toolchainWatch) start() error {
|
||||
for module := range build.LanguageOf {
|
||||
if err := w.ensure(module); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *toolchainWatch) ensure(module string) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.watched[module] {
|
||||
return nil
|
||||
}
|
||||
name := delivery.ToolchainsBucket(module)
|
||||
kv, err := w.js.KeyValue(w.ctx, name)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.set.Put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
|
||||
if w.set.Put(module, e.Key(), e.Value(), !opOf(e)) {
|
||||
w.changed(w.ctx)
|
||||
}
|
||||
})
|
||||
}()
|
||||
w.changed(w.ctx) // what the settle read is new to the builder
|
||||
return nil
|
||||
}
|
||||
|
||||
// delivered is the part of a JetStream message the watch uses.
|
||||
type delivered interface {
|
||||
Subject() string
|
||||
Ack() error
|
||||
Term() error
|
||||
}
|
||||
|
||||
// Reported takes one toolchain.reported event of a build module in: the module is read off the subject,
|
||||
// mesh.stream.<module>.event.toolchain.reported, and its bucket is watched if it is not. A subject of no build module is
|
||||
// terminated; a failure stays unacknowledged, and the server delivers it again.
|
||||
func (w *toolchainWatch) Reported(m delivered) {
|
||||
module, _, _ := strings.Cut(strings.TrimPrefix(m.Subject(), "mesh.stream."), ".event.")
|
||||
if _, known := build.LanguageOf[module]; !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") {
|
||||
w.logger.Printf("%s: not a build module's event, dropped", m.Subject())
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if err := w.ensure(module); err != nil {
|
||||
w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"capacity": "shanks=3,ace=2,g14=0,novox=0"
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
module mesh-delivery-builder
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||
github.com/google/go-tpm v0.9.8 // indirect
|
||||
github.com/klauspost/compress v1.20.0 // indirect
|
||||
github.com/minio/highwayhash v1.0.4 // indirect
|
||||
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
golang.org/x/crypto v0.57.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||
@@ -0,0 +1,88 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the
|
||||
// version the nodes agree on, the ones with room under their build capacity first (places), then the fewest lanes the
|
||||
// builder has running or chosen there, a tie broken by the node's name. A place is reserved on each node chosen, in the
|
||||
// same step as the choice, so builds choosing at once do not choose the same full node while another has room; the
|
||||
// caller takes the place (Places.Acquire) or gives it up (Places.Cancel). A node at its capacity is still chosen when
|
||||
// the others are too (fewer than want have room): its lane waits for a place. A node qualifies only when qualifies is true
|
||||
// of it (online and its build module healthy; nil means every node does). It answers the toolchain all the reporting nodes
|
||||
// agree on, qualifying or not, as the indexer computes it. When nothing is reported, or the nodes report different
|
||||
// versions (so the indexer agreed none), it fails naming the language and why; when no reporting node qualifies it fails
|
||||
// naming the nodes. A node whose build module refuses on its toolchain refuses the step, and the build fails naming it.
|
||||
func ChooseNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, delivery.Toolchain2, error) {
|
||||
tc, none := delivery.AgreedToolchain(entries)
|
||||
if none != "" {
|
||||
return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none)
|
||||
}
|
||||
nodes, err := choose(language, entries, places, qualifies, want)
|
||||
return nodes, tc, err
|
||||
}
|
||||
|
||||
// ChooseAnyNodes is ChooseNodes for artifacts that need no toolchain, such as an archive (ADR 0478 clause 1): the
|
||||
// build module's nodes agree on nothing, so any node of the build module that qualifies is chosen, by capacity and load only.
|
||||
func ChooseAnyNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
|
||||
return choose(language, entries, places, qualifies, want)
|
||||
}
|
||||
|
||||
func choose(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
|
||||
if places == nil {
|
||||
places = &Places{}
|
||||
}
|
||||
nodes := make([]string, 0, len(entries))
|
||||
var reporting []string
|
||||
seen := map[string]bool{}
|
||||
for _, e := range entries {
|
||||
if seen[e.Node] {
|
||||
continue
|
||||
}
|
||||
seen[e.Node] = true
|
||||
reporting = append(reporting, e.Node)
|
||||
if places.CapacityOf(e.Node) > 0 && (qualifies == nil || qualifies(e.Node)) { // a capacity of 0 keeps the builder off a node
|
||||
nodes = append(nodes, e.Node)
|
||||
}
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
sort.Strings(reporting)
|
||||
return nil, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online with a healthy build module and a build capacity", language, strings.Join(reporting, ", "))
|
||||
}
|
||||
return places.choose(nodes, want), nil
|
||||
}
|
||||
|
||||
// DigestsDifferPrefix opens the error of a build whose nodes built an artifact to different digests (ADR 0478 clause 4):
|
||||
// such a key is not built again as a new revision, since the same source on the same toolchains gave two answers and
|
||||
// building again settles nothing; it waits for another fingerprint.
|
||||
const DigestsDifferPrefix = "digests differ:"
|
||||
|
||||
// Agree is the one digest every node built an artifact to (ADR 0478 clause 4), or an error naming each node and its
|
||||
// digest when they differ. One node's digest agrees with itself.
|
||||
func Agree(artifact string, byNode map[string]string) (string, error) {
|
||||
nodes := make([]string, 0, len(byNode))
|
||||
for n := range byNode {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
sort.Strings(nodes)
|
||||
var first string
|
||||
for i, n := range nodes {
|
||||
if i == 0 {
|
||||
first = byNode[n]
|
||||
continue
|
||||
}
|
||||
if byNode[n] != first {
|
||||
msg := fmt.Sprintf("%s the nodes built %s to different digests:", DigestsDifferPrefix, artifact)
|
||||
for _, n := range nodes {
|
||||
msg += fmt.Sprintf(" %s %s;", n, byNode[n])
|
||||
}
|
||||
return "", fmt.Errorf("%s", msg[:len(msg)-1])
|
||||
}
|
||||
}
|
||||
return first, nil
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func tcs(language, version string, nodes ...string) []delivery.NodeToolchain {
|
||||
var out []delivery.NodeToolchain
|
||||
for _, n := range nodes {
|
||||
out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestChooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3
|
||||
entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks")
|
||||
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}), nil, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(nodes) != 2 || nodes[0] != "shanks" || nodes[1] != "ace" {
|
||||
t.Errorf("nodes = %v, want [shanks ace]: fewest steps first, a tie by name", nodes)
|
||||
}
|
||||
if tc.Version != "go1.27.1" || tc.Recipe != "1" || tc.Language != "go" {
|
||||
t.Errorf("toolchain = %+v", tc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4
|
||||
nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2)
|
||||
if err != nil || len(nodes) != 1 || nodes[0] != "novox" {
|
||||
t.Errorf("nodes = %v, err %v, want [novox]", nodes, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4
|
||||
_, _, err := ChooseNodes("typescript", nil, nil, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) {
|
||||
t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport)
|
||||
}
|
||||
entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...)
|
||||
_, _, err = ChooseNodes("go", entries, nil, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") {
|
||||
t.Errorf("disagreeing nodes: error = %v, want both versions named", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
|
||||
d, err := Agree("tool", map[string]string{"ace": digest, "novox": digest})
|
||||
if err != nil || d != digest {
|
||||
t.Errorf("Agree = %q, %v, want %q", d, err, digest)
|
||||
}
|
||||
_, err = Agree("tool", map[string]string{"ace": digest, "novox": "sha256:" + hex64('9')})
|
||||
if err == nil || !strings.Contains(err.Error(), "ace") || !strings.Contains(err.Error(), "novox") || !strings.Contains(err.Error(), digest) || !strings.Contains(err.Error(), hex64('9')) || !strings.Contains(err.Error(), "tool") {
|
||||
t.Errorf("differing digests: error = %v, want the artifact, both nodes and both digests", err)
|
||||
}
|
||||
if d, err := Agree("tool", map[string]string{"ace": digest}); err != nil || d != digest {
|
||||
t.Errorf("one node: %q, %v", d, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3
|
||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox")
|
||||
online := func(n string) bool { return n != "g14" } // a sleeping laptop
|
||||
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 5}), online, 2)
|
||||
if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" {
|
||||
t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err)
|
||||
}
|
||||
_, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") {
|
||||
t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err)
|
||||
}
|
||||
if tc.Version != "go1.27.1" {
|
||||
t.Errorf("none online: the agreed toolchain %+v is still answered", tc)
|
||||
}
|
||||
}
|
||||
|
||||
// placesRunning is Places with these steps counted as running, for a test of the choice.
|
||||
func placesRunning(running map[string]int) *Places {
|
||||
return &Places{running: running, changed: make(chan struct{})}
|
||||
}
|
||||
|
||||
func TestChoosePrefersNodesBelowTheirCapacity(t *testing.T) { // round 3
|
||||
entries := tcs("go", "go1.27.1", "ace", "novox", "shanks")
|
||||
// shanks has capacity 3 with 2 running: room; ace has the default 1 with 1 running: none; novox has none running.
|
||||
p := placesRunning(map[string]int{"shanks": 2, "ace": 1})
|
||||
p.Capacity = map[string]int{"shanks": 3}
|
||||
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
|
||||
if err != nil || len(nodes) != 2 || nodes[0] != "novox" || nodes[1] != "shanks" {
|
||||
t.Errorf("nodes = %v, err %v, want [novox shanks]: the nodes with room, ace at capacity left out", nodes, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ErrConflict is what a Store answers when a Create finds the key or an Update finds another revision than the one given.
|
||||
var ErrConflict = errors.New("the key exists or changed")
|
||||
|
||||
// Store is the builder's bucket (builds): create and compare-and-set, each answering the key's new revision. Only the
|
||||
// builder's credential writes it (ADR 0429 clause 5).
|
||||
type Store interface {
|
||||
Get(ctx context.Context, key string) (value []byte, revision uint64, found bool, err error)
|
||||
Create(ctx context.Context, key string, value []byte) (uint64, error)
|
||||
Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error)
|
||||
// Keys are every key of the bucket.
|
||||
Keys(ctx context.Context) ([]string, error)
|
||||
}
|
||||
|
||||
// Claimer claims fingerprints for the builder instance By (ADR 0429 clause 5: a builder claims a fingerprint by creating
|
||||
// its key and moves it by compare-and-set).
|
||||
type Claimer struct {
|
||||
Store Store
|
||||
By string
|
||||
}
|
||||
|
||||
// Claim is a build the builder holds: its key, the revision the next write must be made at and the value last written.
|
||||
type Claim struct {
|
||||
Key string
|
||||
Rev uint64
|
||||
Build delivery.BuilderBuild
|
||||
}
|
||||
|
||||
// OneNodePrefix opens the Step of a done build that a build module built on one node only, because no second node
|
||||
// qualified (ADR 0478 clause 4); the build modules that did so follow in parentheses, then ": ". Claim reads it to build
|
||||
// the key again, as a new revision, once a second node qualifies.
|
||||
const OneNodePrefix = "one node ("
|
||||
|
||||
// OneNodeModules are the build modules a done build built on one node only, or none when it is not such a build.
|
||||
func OneNodeModules(b delivery.BuilderBuild) []string {
|
||||
if b.State != delivery.BuildDone || !strings.HasPrefix(b.Step, OneNodePrefix) {
|
||||
return nil
|
||||
}
|
||||
list, _, ok := strings.Cut(strings.TrimPrefix(b.Step, OneNodePrefix), "): ")
|
||||
if !ok || list == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Split(list, ", ")
|
||||
}
|
||||
|
||||
// ClaimOptions say what the builder knows about the moment of a claim.
|
||||
type ClaimOptions struct {
|
||||
// TakeOver takes a key found building whoever holds it: the builder has just started and looks at the key for the first
|
||||
// time, and a build left building by a builder that is gone, on this node or another, is nobody's (ADR 0478 clause 6).
|
||||
TakeOver bool
|
||||
// MoreNodes says whether one of these build modules, which built a done key on one node only, has a second node that
|
||||
// qualifies now and the key may be built again (it is not held back after a failed second build, issue 1664). Nil means
|
||||
// none does.
|
||||
MoreNodes func(modules []string) bool
|
||||
}
|
||||
|
||||
// Claim takes the build of module at fingerprint, or answers nil when it is not this builder's to build now:
|
||||
// - no key: it is created building, by this builder; a create that finds the key made meanwhile gets nothing;
|
||||
// - a done key is not taken again (the same source is never built twice), except one built on one node only (OneNodePrefix)
|
||||
// when a second node qualifies now (ClaimOptions.MoreNodes): that is taken as it stands, with the done value, and its
|
||||
// builder writes the new revision when the second build ends (ADR 0478 clause 4);
|
||||
// - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no
|
||||
// builder builds yet (RefusedPrefix, ADR 0478 clause 2) or for digests that differ (DigestsDifferPrefix, clause 4),
|
||||
// which wait for another fingerprint;
|
||||
// - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its
|
||||
// own again when it starts), and so is one left by another builder when ClaimOptions.TakeOver (clause 6).
|
||||
func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string, opt ClaimOptions) (*Claim, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fingerprint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
building := delivery.BuilderBuild{State: delivery.BuildBuilding, By: c.By, Step: "claimed", Manifest: manifest}
|
||||
value, rev, found, err := c.Store.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Create(ctx, key, data) })
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s holds what is not a build: %w", key, err)
|
||||
}
|
||||
switch {
|
||||
case cur.State == delivery.BuildDone:
|
||||
if modules := OneNodeModules(cur); len(modules) > 0 && opt.MoreNodes != nil && opt.MoreNodes(modules) {
|
||||
return &Claim{Key: key, Rev: rev, Build: cur}, nil
|
||||
}
|
||||
return nil, nil
|
||||
case cur.State == delivery.BuildFailed && (strings.HasPrefix(cur.Error, RefusedPrefix) || strings.HasPrefix(cur.Error, DigestsDifferPrefix)):
|
||||
return nil, nil
|
||||
case cur.State == delivery.BuildFailed:
|
||||
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
|
||||
case cur.By == c.By || opt.TakeOver:
|
||||
return &Claim{Key: key, Rev: rev, Build: cur}, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (c Claimer) write(_ context.Context, key string, b delivery.BuilderBuild, put func([]byte) (uint64, error)) (*Claim, error) {
|
||||
data, err := delivery.EncodeBuilderBuild(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rev, err := put(data)
|
||||
if errors.Is(err, ErrConflict) {
|
||||
return nil, nil // another builder made it first
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
b.V = delivery.Version
|
||||
return &Claim{Key: key, Rev: rev, Build: b}, nil
|
||||
}
|
||||
|
||||
// Finish moves the claimed key to b by compare-and-set at the claim's revision, and writes nothing when b is the value
|
||||
// the key holds. A key that changed meanwhile is ErrConflict: the claim is no longer this builder's.
|
||||
func (c Claimer) Finish(ctx context.Context, cl *Claim, b delivery.BuilderBuild) error {
|
||||
b.V = delivery.Version
|
||||
if delivery.SameBuilderBuild(cl.Build, b) {
|
||||
return nil
|
||||
}
|
||||
data, err := delivery.EncodeBuilderBuild(b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rev, err := c.Store.Update(ctx, cl.Key, data, cl.Rev)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", cl.Key, err)
|
||||
}
|
||||
cl.Rev, cl.Build = rev, b
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// memStore is a bucket's create and compare-and-set in memory, with a revision per write.
|
||||
type memStore struct {
|
||||
mu sync.Mutex
|
||||
vals map[string][]byte
|
||||
revs map[string]uint64
|
||||
next uint64
|
||||
writes int
|
||||
// before runs once ahead of the next write, so a test changes the key under the writer.
|
||||
before func()
|
||||
}
|
||||
|
||||
func newMem() *memStore { return &memStore{vals: map[string][]byte{}, revs: map[string]uint64{}} }
|
||||
|
||||
func (m *memStore) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
v, ok := m.vals[key]
|
||||
return v, m.revs[key], ok, nil
|
||||
}
|
||||
|
||||
func (m *memStore) Keys(_ context.Context) ([]string, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var keys []string
|
||||
for k := range m.vals {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (m *memStore) hook() {
|
||||
if h := m.before; h != nil {
|
||||
m.before = nil
|
||||
h()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *memStore) put(key string, value []byte) uint64 {
|
||||
m.next++
|
||||
m.writes++
|
||||
m.vals[key], m.revs[key] = value, m.next
|
||||
return m.next
|
||||
}
|
||||
|
||||
func (m *memStore) Create(_ context.Context, key string, value []byte) (uint64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.hook()
|
||||
if _, ok := m.vals[key]; ok {
|
||||
return 0, ErrConflict
|
||||
}
|
||||
return m.put(key, value), nil
|
||||
}
|
||||
|
||||
func (m *memStore) Update(_ context.Context, key string, value []byte, rev uint64) (uint64, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.hook()
|
||||
if _, ok := m.vals[key]; !ok || m.revs[key] != rev {
|
||||
return 0, ErrConflict
|
||||
}
|
||||
return m.put(key, value), nil
|
||||
}
|
||||
|
||||
const (
|
||||
fp1 = "src2:1111111111111111111111111111111111111111111111111111111111111111"
|
||||
digest = "sha256:2222222222222222222222222222222222222222222222222222222222222222"
|
||||
)
|
||||
|
||||
func seed(t *testing.T, st *memStore, b delivery.BuilderBuild) string {
|
||||
t.Helper()
|
||||
key, err := delivery.BuilderBuildKey("m", fp1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := delivery.EncodeBuilderBuild(b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st.put(key, data)
|
||||
return key
|
||||
}
|
||||
|
||||
func valueOf(t *testing.T, st *memStore, key string) delivery.BuilderBuild {
|
||||
t.Helper()
|
||||
b, err := delivery.DecodeBuilderBuild(st.vals[key])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestClaimCreatesTheKeyBuildingAndASecondBuilderGetsNothing(t *testing.T) { // 1607/AC3
|
||||
st := newMem()
|
||||
a, b := Claimer{Store: st, By: "novox.a"}, Claimer{Store: st, By: "novox.b"}
|
||||
got, err := a.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{})
|
||||
if err != nil || got == nil {
|
||||
t.Fatalf("first claim = %v, %v, want a claim", got, err)
|
||||
}
|
||||
if v := valueOf(t, st, got.Key); v.State != delivery.BuildBuilding || v.By != "novox.a" || v.Manifest != hex64('a') {
|
||||
t.Errorf("the key holds %+v, want building by novox.a with the manifest digest", v)
|
||||
}
|
||||
if second, err := b.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{}); err != nil || second != nil {
|
||||
t.Errorf("second builder's claim = %v, %v, want none: the same source is never built twice", second, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimLosingTheCreateRaceGetsNothing(t *testing.T) { // 1607/AC3
|
||||
st := newMem()
|
||||
st.before = func() { seed(t, st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.b"}) }
|
||||
got, err := Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, "", ClaimOptions{})
|
||||
if err != nil || got != nil {
|
||||
t.Errorf("claim = %v, %v, want none: the create failed because the key exists", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADoneKeyIsNeverTakenAgain(t *testing.T) { // 1607/AC3
|
||||
st := newMem()
|
||||
seed(t, st, delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Digest: digest}}})
|
||||
writes := st.writes
|
||||
got, err := Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{})
|
||||
if err != nil || got != nil || st.writes != writes {
|
||||
t.Errorf("claim = %v, %v, writes %d (was %d), want none and no write", got, err, st.writes, writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedKeyIsTakenAgainAsANewRevisionExceptARefusedKind(t *testing.T) { // 1607/AC3, ADR 0478 clause 2
|
||||
st := newMem()
|
||||
key := seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: "the step failed"})
|
||||
rev := st.revs[key]
|
||||
got, err := Claimer{Store: st, By: "novox.a"}.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{})
|
||||
if err != nil || got == nil || st.revs[key] <= rev {
|
||||
t.Fatalf("claim = %v, %v, revision %d (was %d), want a claim as a new revision of the same key", got, err, st.revs[key], rev)
|
||||
}
|
||||
if v := valueOf(t, st, key); v.State != delivery.BuildBuilding || v.Error != "" {
|
||||
t.Errorf("the key holds %+v, want building with no error", v)
|
||||
}
|
||||
|
||||
st = newMem()
|
||||
key = seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, By: "novox.a", Error: RefusedPrefix + " image (artifact x): build-agent builds it"})
|
||||
writes := st.writes
|
||||
if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got != nil || st.writes != writes {
|
||||
t.Errorf("a refused kind: claim = %v, %v, writes %d (was %d), want none and no write", got, err, st.writes, writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinishMovesTheKeyByCompareAndSetAndWritesOnlyADifferentValue(t *testing.T) { // 1607/AC3
|
||||
st := newMem()
|
||||
c := Claimer{Store: st, By: "novox.a"}
|
||||
got, _ := c.Claim(context.Background(), "m", fp1, hex64('a'), ClaimOptions{})
|
||||
done := delivery.BuilderBuild{State: delivery.BuildDone, By: "novox.a", Manifest: hex64('a'), Artifacts: []delivery.BuiltArtifact{{Name: "x", Kind: "archive", Digest: digest}}}
|
||||
if err := c.Finish(context.Background(), got, done); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := valueOf(t, st, got.Key); v.State != delivery.BuildDone || v.Artifacts[0].Digest != digest {
|
||||
t.Errorf("the key holds %+v", v)
|
||||
}
|
||||
writes := st.writes
|
||||
if err := c.Finish(context.Background(), got, done); err != nil && !errors.Is(err, ErrConflict) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if st.writes != writes {
|
||||
t.Errorf("the same value was written again (%d writes, was %d)", st.writes, writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinishAfterTheKeyChangedIsAConflictNotAnOverwrite(t *testing.T) { // 1607/AC3
|
||||
st := newMem()
|
||||
c := Claimer{Store: st, By: "novox.a"}
|
||||
got, _ := c.Claim(context.Background(), "m", fp1, "", ClaimOptions{})
|
||||
st.put(got.Key, []byte(`{"v":1,"state":"failed","error":"someone else"}`))
|
||||
err := c.Finish(context.Background(), got, delivery.BuilderBuild{State: delivery.BuildFailed, Error: "mine"})
|
||||
if !errors.Is(err, ErrConflict) {
|
||||
t.Errorf("Finish = %v, want ErrConflict", err)
|
||||
}
|
||||
}
|
||||
|
||||
func hex64(c byte) string {
|
||||
b := make([]byte, 64)
|
||||
for i := range b {
|
||||
b[i] = c
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// 1664: a done key built on one node is not built again, over and over, while the second node's build keeps failing.
|
||||
func TestASecondNodeBuildThatFailsIsNotRetriedUntilTheNodesChange(t *testing.T) { // issue 1664
|
||||
r := newRig(t, goBundle)
|
||||
r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies
|
||||
first := r.consider(t, "modules/m")
|
||||
r.bm.fails["ace/build_go_fetch"] = "ace is out of disk"
|
||||
r.pres.Put("ace", presenceValue(t, true), false)
|
||||
r.b.NodesChanged(context.Background()) // ace came online: one more try
|
||||
r.b.Wait()
|
||||
tries := len(r.bm.callsOf("build_go_fetch"))
|
||||
if tries < 2 {
|
||||
t.Fatalf("%d fetches: the second node was never tried", tries)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) {
|
||||
t.Fatalf("the done key changed: %+v", v)
|
||||
}
|
||||
// Nothing about the nodes changed: a modules value, a Reconsider and a start of looking again build nothing.
|
||||
for i := 0; i < 3; i++ {
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
}
|
||||
if n := len(r.bm.callsOf("build_go_fetch")); n != tries {
|
||||
t.Errorf("%d fetches after looking again, was %d: the failed second build was retried in a loop", n, tries)
|
||||
}
|
||||
// A node changing (ace goes and comes back) is a reason to try again, and ace now builds.
|
||||
delete(r.bm.fails, "ace/build_go_fetch")
|
||||
r.pres.Put("ace", presenceValue(t, false), false)
|
||||
r.b.NodesChanged(context.Background())
|
||||
r.pres.Put("ace", presenceValue(t, true), false)
|
||||
r.b.NodesChanged(context.Background())
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") {
|
||||
t.Errorf("build = %+v, want it built on both nodes once ace could", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 1660: at start the builder closes a key left building for a fingerprint the module no longer has.
|
||||
func TestAKeyLeftBuildingForAFingerprintTheModuleNoLongerHasIsClosedAtStart(t *testing.T) { // issue 1660
|
||||
r := newRig(t, goBundle)
|
||||
stale := seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD}) // m at fp1
|
||||
other, _ := delivery.BuilderBuildKey("gone", fp1)
|
||||
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD})
|
||||
r.st.put(other, data)
|
||||
// The module m has moved on to another fingerprint.
|
||||
moved := delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'),
|
||||
FP2: "src2:" + hex64('7'), Manifest: manifestD, Path: "modules/m", Toolchains: agreedToolchains}
|
||||
enc, _ := delivery.EncodeModule(moved)
|
||||
r.b.Module(context.Background(), "m", enc)
|
||||
r.b.Wait()
|
||||
if err := r.b.Sweep(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{stale, other} {
|
||||
v := valueOf(t, r.st, k)
|
||||
if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "no longer") || !strings.Contains(v.Error, "ace.builder") {
|
||||
t.Errorf("%s = %+v, want it closed as failed, naming why and who left it", k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepLeavesAKeyOfTheModulesCurrentFingerprintForTheBuildToTakeOver(t *testing.T) { // issue 1660
|
||||
r := newRig(t, goBundle)
|
||||
seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD})
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // takes it over and builds it
|
||||
r.b.Wait()
|
||||
if err := r.b.Sweep(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.By != "novox.builder" {
|
||||
t.Errorf("build = %+v, want done by novox.builder", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// HaveSet holds what each node's node-engine says of each module on it (node-engine_have, key `<node>.<module>`): the
|
||||
// builder reads it to know whether a node's build module is running and healthy, since a build module whose installed
|
||||
// toolchain is not the one its recipe builds with is unhealthy (mesh-build-go build_go_health) and its steps would
|
||||
// refuse (ADR 0478 clause 3). A module the node-engine has written no key of is not healthy. Safe for use from the watch
|
||||
// and from a build at once.
|
||||
type HaveSet struct {
|
||||
mu sync.Mutex
|
||||
healthy map[string]bool // by `<node>.<module>`
|
||||
}
|
||||
|
||||
// Put takes a change of the have bucket in and answers whether a module's healthy state changed. A node's own key
|
||||
// (`<node>.node`) and a value that is not a have are left out.
|
||||
func (s *HaveSet) Put(key string, value []byte, deleted bool) bool {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
return false
|
||||
}
|
||||
ok := false
|
||||
if !deleted {
|
||||
h, err := delivery.DecodeHave(value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
ok = h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
|
||||
}
|
||||
k := node + "." + module
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.healthy[k] == ok {
|
||||
return false
|
||||
}
|
||||
if s.healthy == nil {
|
||||
s.healthy = map[string]bool{}
|
||||
}
|
||||
if ok {
|
||||
s.healthy[k] = true
|
||||
} else {
|
||||
delete(s.healthy, k)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Healthy is whether the module on the node was last seen running and healthy.
|
||||
func (s *HaveSet) Healthy(node, module string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.healthy[node+"."+module]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// Manifests reads a module's manifest by its digest, the key of the indexer's manifests bucket.
|
||||
type Manifests interface {
|
||||
Manifest(ctx context.Context, digest string) (catalogue.Manifest, error)
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// DefaultCapacity is the build capacity of a node that has no setting: one build step at a time.
|
||||
const DefaultCapacity = 1
|
||||
|
||||
// ParseCapacity reads the builder's capacity setting: `node=steps` pairs separated by commas, such as
|
||||
// `shanks=3,ace=2`. A node without a pair has DefaultCapacity. A pair with no node, a count that is not a whole number of
|
||||
// at least 0, and a node named twice are refused. A count of 0 keeps the builder off the node: it is never chosen.
|
||||
func ParseCapacity(s string) (map[string]int, error) {
|
||||
out := map[string]int{}
|
||||
for _, pair := range strings.Split(s, ",") {
|
||||
pair = strings.TrimSpace(pair)
|
||||
if pair == "" {
|
||||
continue
|
||||
}
|
||||
node, count, ok := strings.Cut(pair, "=")
|
||||
node, count = strings.TrimSpace(node), strings.TrimSpace(count)
|
||||
n, err := strconv.Atoi(count)
|
||||
if !ok || node == "" || err != nil || n < 0 {
|
||||
return nil, fmt.Errorf("the capacity %q is not <node>=<steps at once, 0 or more>", pair)
|
||||
}
|
||||
if _, dup := out[node]; dup {
|
||||
return nil, fmt.Errorf("the capacity names %s twice", node)
|
||||
}
|
||||
out[node] = n
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Places counts the build steps the builder runs on each node and holds a lane back while its node is at its
|
||||
// capacity (the operator's decision of 2026-10-21; ADR 0478 clause 3). A lane is the fetch and the builds of one build
|
||||
// module on one node, run one step at a time, so a lane holds one place while it runs. A node is chosen for a lane by
|
||||
// reserving a place on it at the moment of the choice (choose), so that builds choosing at once see each other's
|
||||
// choices and none waits on a full node while another has room. A lane waits for its place by watching the ends of the
|
||||
// other lanes, not by a timer: ending a lane wakes the waiting ones.
|
||||
type Places struct {
|
||||
// Capacity is the number of steps at once by node; a node not in it has DefaultCapacity.
|
||||
Capacity map[string]int
|
||||
|
||||
mu sync.Mutex
|
||||
running map[string]int // lanes holding a place
|
||||
reserved map[string]int // lanes chosen for the node that do not hold a place yet
|
||||
changed chan struct{} // closed, and replaced, each time a lane ends
|
||||
}
|
||||
|
||||
// CapacityOf is how many steps the builder runs on the node at once: DefaultCapacity for a node the setting does not
|
||||
// name, and 0 for one it names so, where the builder never builds.
|
||||
func (p *Places) CapacityOf(node string) int {
|
||||
if n, ok := p.Capacity[node]; ok && n >= 0 {
|
||||
return n
|
||||
}
|
||||
return DefaultCapacity
|
||||
}
|
||||
|
||||
// Running is the steps in flight now, by node.
|
||||
func (p *Places) Running() map[string]int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make(map[string]int, len(p.running))
|
||||
for n, c := range p.running {
|
||||
if c > 0 {
|
||||
out[n] = c
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Reserved is the lanes chosen for a node that hold no place yet, by node.
|
||||
func (p *Places) Reserved() map[string]int {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
out := make(map[string]int, len(p.reserved))
|
||||
for n, c := range p.reserved {
|
||||
if c > 0 {
|
||||
out[n] = c
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (p *Places) init() {
|
||||
if p.running == nil {
|
||||
p.running = map[string]int{}
|
||||
}
|
||||
if p.reserved == nil {
|
||||
p.reserved = map[string]int{}
|
||||
}
|
||||
if p.changed == nil {
|
||||
p.changed = make(chan struct{})
|
||||
}
|
||||
}
|
||||
|
||||
// Acquire takes a place on the node, waiting while the node is at its capacity, and answers the function that gives it
|
||||
// back. A place reserved for the node by a choice is the one taken. It returns an error when ctx ends first, and then
|
||||
// gives up the reservation too.
|
||||
func (p *Places) Acquire(ctx context.Context, node string) (release func(), err error) {
|
||||
for {
|
||||
p.mu.Lock()
|
||||
p.init()
|
||||
if p.running[node] < p.CapacityOf(node) {
|
||||
p.running[node]++
|
||||
if p.reserved[node] > 0 {
|
||||
p.reserved[node]--
|
||||
}
|
||||
p.mu.Unlock()
|
||||
var once sync.Once
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
p.mu.Lock()
|
||||
p.running[node]--
|
||||
close(p.changed)
|
||||
p.changed = make(chan struct{})
|
||||
p.mu.Unlock()
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
wait := p.changed
|
||||
p.mu.Unlock()
|
||||
select {
|
||||
case <-wait:
|
||||
case <-ctx.Done():
|
||||
p.Cancel(node)
|
||||
return nil, fmt.Errorf("waiting for a place on %s: %w", node, ctx.Err())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Cancel gives up a place reserved for the node that will not be taken.
|
||||
func (p *Places) Cancel(node string) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.init()
|
||||
if p.reserved[node] > 0 {
|
||||
p.reserved[node]--
|
||||
}
|
||||
}
|
||||
|
||||
// choose orders the nodes (those with room first, counting the lanes running and the lanes reserved, then the fewest of
|
||||
// them, then by name), keeps the first want and reserves a place on each, all at once.
|
||||
func (p *Places) choose(nodes []string, want int) []string {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.init()
|
||||
load := func(n string) int { return p.running[n] + p.reserved[n] }
|
||||
sort.SliceStable(nodes, func(i, j int) bool {
|
||||
ri, rj := load(nodes[i]) < p.CapacityOf(nodes[i]), load(nodes[j]) < p.CapacityOf(nodes[j])
|
||||
if ri != rj {
|
||||
return ri
|
||||
}
|
||||
if load(nodes[i]) != load(nodes[j]) {
|
||||
return load(nodes[i]) < load(nodes[j])
|
||||
}
|
||||
return nodes[i] < nodes[j]
|
||||
})
|
||||
if len(nodes) > want {
|
||||
nodes = nodes[:want]
|
||||
}
|
||||
for _, n := range nodes {
|
||||
p.reserved[n]++
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
func TestParseCapacityReadsNodeEqualsCountPairs(t *testing.T) {
|
||||
c, err := ParseCapacity(" shanks=3, ace=2 ")
|
||||
if err != nil || c["shanks"] != 3 || c["ace"] != 2 || len(c) != 2 {
|
||||
t.Errorf("capacity = %v, %v", c, err)
|
||||
}
|
||||
if c, err := ParseCapacity(""); err != nil || len(c) != 0 {
|
||||
t.Errorf("empty: %v, %v", c, err)
|
||||
}
|
||||
for _, bad := range []string{"shanks", "shanks=x", "=2", "a=1,a=2", "shanks=-1"} {
|
||||
if _, err := ParseCapacity(bad); err == nil {
|
||||
t.Errorf("%q accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func acquireAsync(p *Places, node string) chan func() {
|
||||
got := make(chan func(), 1)
|
||||
go func() {
|
||||
release, err := p.Acquire(context.Background(), node)
|
||||
if err != nil {
|
||||
close(got)
|
||||
return
|
||||
}
|
||||
got <- release
|
||||
}()
|
||||
return got
|
||||
}
|
||||
|
||||
func TestThreeStepsOnANodeOfCapacityTwoRunTwoAtOnceAndTheThirdStartsWhenOneEnds(t *testing.T) { // round 3
|
||||
p := &Places{Capacity: map[string]int{"ace": 2}}
|
||||
a, b := acquireAsync(p, "ace"), acquireAsync(p, "ace")
|
||||
ra, rb := <-a, <-b
|
||||
third := acquireAsync(p, "ace")
|
||||
select {
|
||||
case <-third:
|
||||
t.Fatal("a third step started on a node of capacity 2 with two running")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
if p.Running()["ace"] != 2 {
|
||||
t.Errorf("running = %v, want 2 on ace", p.Running())
|
||||
}
|
||||
ra()
|
||||
select {
|
||||
case r := <-third:
|
||||
r()
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the third step did not start when one ended")
|
||||
}
|
||||
rb()
|
||||
}
|
||||
|
||||
func TestANodeWithNoSettingRunsOneStepAtATime(t *testing.T) { // round 3
|
||||
p := &Places{Capacity: map[string]int{"shanks": 3}}
|
||||
first := <-acquireAsync(p, "g14")
|
||||
second := acquireAsync(p, "g14")
|
||||
select {
|
||||
case <-second:
|
||||
t.Fatal("two steps on a node without a setting")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
first()
|
||||
r := <-second
|
||||
r()
|
||||
if p.CapacityOf("g14") != 1 || p.CapacityOf("shanks") != 3 {
|
||||
t.Errorf("capacities = %d, %d", p.CapacityOf("g14"), p.CapacityOf("shanks"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWaitingStepStopsWhenItsContextEnds(t *testing.T) { // round 3
|
||||
p := &Places{}
|
||||
r := <-acquireAsync(p, "ace")
|
||||
defer r()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan error, 1)
|
||||
go func() { _, err := p.Acquire(ctx, "ace"); done <- err }()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Error("Acquire returned no error for a cancelled wait")
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("a cancelled wait did not return")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStepsOfSeveralBuildsNeverExceedANodesCapacity(t *testing.T) { // round 3, through the builder
|
||||
r := newRig(t, catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"})
|
||||
r.b.Places = &Places{Capacity: map[string]int{"ace": 2}} // novox has none: one at a time
|
||||
var now, most atomic.Int32
|
||||
var mu sync.Mutex
|
||||
perNode := map[string]int{}
|
||||
r.bm.steps.during = func(key string) func() {
|
||||
node, _, _ := strings.Cut(key, ".")
|
||||
mu.Lock()
|
||||
perNode[node]++
|
||||
if int32(perNode[node]) > most.Load() && node == "ace" {
|
||||
most.Store(int32(perNode[node]))
|
||||
}
|
||||
if node == "novox" && perNode[node] > 1 {
|
||||
t.Errorf("%d steps at once on novox, which has capacity 1", perNode[node])
|
||||
}
|
||||
mu.Unlock()
|
||||
now.Add(1)
|
||||
time.Sleep(15 * time.Millisecond)
|
||||
return func() { mu.Lock(); perNode[node]--; mu.Unlock(); now.Add(-1) }
|
||||
}
|
||||
for _, m := range []string{"m1", "m2", "m3", "m4"} {
|
||||
r.b.Module(context.Background(), m, moduleValue(t, "modules/"+m))
|
||||
}
|
||||
r.b.Wait()
|
||||
if most.Load() > 2 {
|
||||
t.Errorf("%d steps at once on ace, which has capacity 2", most.Load())
|
||||
}
|
||||
if most.Load() < 2 {
|
||||
t.Errorf("at most %d steps at once on ace: the capacity was not used", most.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoNodeAgreementStillTakesTwoDifferentNodesWhenCapacitiesAreSet(t *testing.T) { // round 3, ADR 0478 clause 4
|
||||
r := newRig(t, catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"})
|
||||
r.b.Places = &Places{Capacity: map[string]int{"ace": 2, "novox": 3}}
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != "done" || !strings.Contains(v.Step, "ace and novox") {
|
||||
t.Errorf("build = %+v, want done on ace and novox", v)
|
||||
}
|
||||
}
|
||||
|
||||
// The operator's decision of 2026-10-21 13:40: the builder builds on shanks and ace only; a capacity of 0 keeps it off a node.
|
||||
func TestACapacityOfZeroKeepsTheBuilderOffTheNodeAndANodeNotNamedKeepsOne(t *testing.T) { // issue 1661
|
||||
c, err := ParseCapacity("shanks=3,ace=2,g14=0,novox=0")
|
||||
if err != nil || c["g14"] != 0 || c["novox"] != 0 || c["shanks"] != 3 || len(c) != 4 {
|
||||
t.Fatalf("capacity = %v, %v", c, err)
|
||||
}
|
||||
p := &Places{Capacity: c}
|
||||
if p.CapacityOf("g14") != 0 || p.CapacityOf("novox") != 0 || p.CapacityOf("ace") != 2 || p.CapacityOf("other") != DefaultCapacity {
|
||||
t.Errorf("capacities = %d %d %d %d, want 0 0 2 and the default for a node not named", p.CapacityOf("g14"), p.CapacityOf("novox"), p.CapacityOf("ace"), p.CapacityOf("other"))
|
||||
}
|
||||
if _, err := ParseCapacity("shanks=-1"); err == nil {
|
||||
t.Error("a negative capacity accepted")
|
||||
}
|
||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks", "other")
|
||||
for i := 0; i < 6; i++ {
|
||||
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n == "g14" || n == "novox" {
|
||||
t.Fatalf("choice %d: %v includes a node of capacity 0", i, nodes)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only nodes of capacity 0 report: nothing qualifies, and the build says why.
|
||||
_, _, err = ChooseNodes("go", tcs("go", "go1.27.1", "g14", "novox"), p, nil, 2)
|
||||
if err == nil || !strings.Contains(err.Error(), "capacity") {
|
||||
t.Errorf("only capacity-0 nodes: error = %v, want it to name the capacity", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildNeverRunsOnANodeOfCapacityZero(t *testing.T) { // issue 1661
|
||||
r := newRig(t, catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}) // ace and novox report
|
||||
r.b.Places = &Places{Capacity: map[string]int{"novox": 0}}
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != "done" || !strings.Contains(v.Step, "on one node, ace") {
|
||||
t.Fatalf("build = %+v, want done on ace alone", v)
|
||||
}
|
||||
for _, c := range r.bm.calls {
|
||||
if c.node == "novox" {
|
||||
t.Errorf("a step was called on novox, whose capacity is 0: %+v", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
// Package build is the builder's logic (novox/hq ADR 0453 clause 4, ADR 0478), apart from the bus: what a manifest asks
|
||||
// to be built and where, the claim of a build key, the choice of build nodes, the build modules' long steps and the run of
|
||||
// one build. The bus is reached through the small interfaces here, so each part is tested with fakes.
|
||||
package build
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The build modules a task is given to, and the language whose toolchain each reports (the indexer's ToolchainModules).
|
||||
const (
|
||||
GoBuildModule = "mesh-build-go"
|
||||
NodeBuildModule = "mesh-build-node"
|
||||
)
|
||||
|
||||
// RefusedPrefix opens the error of a build key that is failed for a kind no builder builds yet (ADR 0478 clause 2): such
|
||||
// a key is not built again as a new revision.
|
||||
const RefusedPrefix = "refused kind:"
|
||||
|
||||
// Task is one artifact to build: its name and kind in the manifest, the build module that builds it and the language
|
||||
// of the toolchain it is built with.
|
||||
type Task struct {
|
||||
Artifact string
|
||||
Kind string
|
||||
Module string
|
||||
Language string
|
||||
}
|
||||
|
||||
// Plan is what a manifest asks the builder to build, by artifact name.
|
||||
type Plan struct{ Tasks []Task }
|
||||
|
||||
// Modules are the build modules the plan calls, sorted.
|
||||
func (p Plan) Modules() []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, t := range p.Tasks {
|
||||
if !seen[t.Module] {
|
||||
seen[t.Module] = true
|
||||
out = append(out, t.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// NeedsToolchain is whether any task the build module builds is built with a toolchain: a bundle is, an archive is not
|
||||
// (the indexer gives an archive no language, so a fingerprint names no toolchain for it; ADR 0478 clause 1).
|
||||
func (p Plan) NeedsToolchain(module string) bool {
|
||||
for _, t := range p.Tasks {
|
||||
if t.Module == module && t.Kind != catalogue.ArtifactArchive {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Refused is the error of a plan the builder does not build: an artifact of a kind (or of a language) it has no build
|
||||
// module for.
|
||||
type Refused struct{ Artifact, What string }
|
||||
|
||||
func (r *Refused) Error() string {
|
||||
return fmt.Sprintf("%s %s (artifact %s): the builder builds Go and TypeScript bundles and archives; build-agent builds it", RefusedPrefix, r.What, r.Artifact)
|
||||
}
|
||||
|
||||
// PlanOf is the plan of a manifest (ADR 0478 clauses 1 and 2): a Go bundle and an archive are built by the Go build
|
||||
// module, a TypeScript bundle by the Node one, each in the order of the artifacts' names. A manifest with an image,
|
||||
// upstream or package artifact, or a bundle of another language, is refused as a whole, naming the first such artifact
|
||||
// by name. A manifest with no artifact has an empty plan.
|
||||
func PlanOf(m catalogue.Manifest) (Plan, error) {
|
||||
if m.Build == nil {
|
||||
return Plan{}, nil
|
||||
}
|
||||
artifacts := append([]catalogue.Artifact(nil), m.Build.Artifacts...)
|
||||
sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name })
|
||||
var p Plan
|
||||
for _, a := range artifacts {
|
||||
switch {
|
||||
case a.Kind == catalogue.ArtifactArchive:
|
||||
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: GoBuildModule, Language: "go"})
|
||||
case a.Kind == catalogue.ArtifactBundle && a.Language == "go":
|
||||
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: GoBuildModule, Language: "go"})
|
||||
case a.Kind == catalogue.ArtifactBundle && a.Language == "typescript":
|
||||
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: NodeBuildModule, Language: "typescript"})
|
||||
case a.Kind == catalogue.ArtifactBundle:
|
||||
return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("bundle of language %q", a.Language)}
|
||||
default:
|
||||
return Plan{}, &Refused{Artifact: a.Name, What: a.Kind}
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// manifestOf is a manifest holding these artifacts: the plan reads only their name, kind and language.
|
||||
func manifestOf(artifacts ...catalogue.Artifact) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}
|
||||
}
|
||||
|
||||
func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2
|
||||
m := manifestOf(
|
||||
catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"},
|
||||
catalogue.Artifact{Name: "web", Kind: "bundle", Language: "typescript"},
|
||||
catalogue.Artifact{Name: "files", Kind: "archive"})
|
||||
p, err := PlanOf(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []Task{
|
||||
{Artifact: "files", Kind: "archive", Module: GoBuildModule, Language: "go"},
|
||||
{Artifact: "tool", Kind: "bundle", Module: GoBuildModule, Language: "go"},
|
||||
{Artifact: "web", Kind: "bundle", Module: NodeBuildModule, Language: "typescript"},
|
||||
}
|
||||
if len(p.Tasks) != len(want) {
|
||||
t.Fatalf("tasks = %+v, want %+v", p.Tasks, want)
|
||||
}
|
||||
for i := range want {
|
||||
if p.Tasks[i] != want[i] {
|
||||
t.Errorf("task %d = %+v, want %+v", i, p.Tasks[i], want[i])
|
||||
}
|
||||
}
|
||||
if modules := p.Modules(); len(modules) != 2 || modules[0] != GoBuildModule || modules[1] != NodeBuildModule {
|
||||
t.Errorf("modules = %v", modules)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanRefusesImageUpstreamAndPackageByName(t *testing.T) { // 1607/AC2, ADR 0478 clause 2
|
||||
for _, c := range []struct {
|
||||
artifact catalogue.Artifact
|
||||
kind string
|
||||
}{
|
||||
{catalogue.Artifact{Name: "img", Kind: "image"}, "image"},
|
||||
{catalogue.Artifact{Name: "up", Kind: "upstream"}, "upstream"},
|
||||
{catalogue.Artifact{Name: "pkg", Kind: "package", Language: "typescript"}, "package"},
|
||||
} {
|
||||
m := manifestOf(catalogue.Artifact{Name: "ok", Kind: "archive"}, c.artifact)
|
||||
p, err := PlanOf(m)
|
||||
if err == nil {
|
||||
t.Errorf("%s: plan %+v, want a refusal", c.kind, p)
|
||||
continue
|
||||
}
|
||||
var r *Refused
|
||||
if !asRefused(err, &r) || !strings.HasPrefix(err.Error(), RefusedPrefix) || !strings.Contains(err.Error(), c.kind) || !strings.Contains(err.Error(), "build-agent") {
|
||||
t.Errorf("%s: error = %v, want it to open %q and name the kind and build-agent", c.kind, err, RefusedPrefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanRefusesABundleOfAnotherLanguageAndAModuleWithNoArtifact(t *testing.T) {
|
||||
if _, err := PlanOf(manifestOf(catalogue.Artifact{Name: "x", Kind: "bundle", Language: "python"})); err == nil || !strings.HasPrefix(err.Error(), RefusedPrefix) {
|
||||
t.Errorf("a python bundle: error = %v, want a refusal", err)
|
||||
}
|
||||
p, err := PlanOf(manifestOf())
|
||||
if err != nil || len(p.Tasks) != 0 {
|
||||
t.Errorf("no artifact: plan %+v error %v, want an empty plan", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
func asRefused(err error, target **Refused) bool { return errors.As(err, target) }
|
||||
@@ -0,0 +1,53 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key `<node>`, the
|
||||
// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the
|
||||
// watchman has no key of is not online. Safe for use from the watch and from a build at once.
|
||||
type PresenceSet struct {
|
||||
mu sync.Mutex
|
||||
online map[string]bool
|
||||
}
|
||||
|
||||
// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a
|
||||
// value that is not a presence are left out.
|
||||
func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool {
|
||||
node, part, _, err := delivery.ParsePresenceKey(key)
|
||||
if err != nil || part != "" {
|
||||
return false
|
||||
}
|
||||
on := false
|
||||
if !deleted {
|
||||
p, err := delivery.DecodePresence(value)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
on = p.Online
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.online[node] == on {
|
||||
return false
|
||||
}
|
||||
if s.online == nil {
|
||||
s.online = map[string]bool{}
|
||||
}
|
||||
if on {
|
||||
s.online[node] = true
|
||||
} else {
|
||||
delete(s.online, node)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Online is whether the watchman last saw the node online.
|
||||
func (s *PresenceSet) Online(node string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.online[node]
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func presenceValue(t *testing.T, online bool) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodePresence(delivery.Presence{Online: online})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3
|
||||
var s PresenceSet
|
||||
if s.Online("ace") {
|
||||
t.Error("a node the watchman never wrote is online")
|
||||
}
|
||||
if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") {
|
||||
t.Error("ace online was not taken in")
|
||||
}
|
||||
if s.Put("ace", presenceValue(t, true), false) {
|
||||
t.Error("the same presence again counted as a change")
|
||||
}
|
||||
if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") {
|
||||
t.Error("ace going offline was not taken in")
|
||||
}
|
||||
s.Put("g14", presenceValue(t, true), false)
|
||||
if !s.Put("g14", nil, true) || s.Online("g14") {
|
||||
t.Error("a deleted key is still online")
|
||||
}
|
||||
// A part's instance (part.<part>.<instance>) is not a node.
|
||||
if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") {
|
||||
t.Error("a part's key was taken for a node")
|
||||
}
|
||||
if s.Put("novox", []byte("junk"), false) {
|
||||
t.Error("junk counted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func haveValue(t *testing.T, phase delivery.Phase, health delivery.Health) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeHave(delivery.Have{Phase: phase, Health: health})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// 1. A module of an archive only: the real indexer gives an archive no language (inputs.go languagesOf), so its
|
||||
// fingerprint names no toolchain, and the builder builds it on the build module's nodes with no toolchain agreed.
|
||||
func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *testing.T) { // review round 4, 1
|
||||
r := newRig(t, archiveA)
|
||||
v := func() delivery.BuilderBuild {
|
||||
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
|
||||
r.b.Wait()
|
||||
if _, ok := r.st.vals[r.key]; !ok {
|
||||
t.Fatal("no build key was written for the archive-only module")
|
||||
}
|
||||
return valueOf(t, r.st, r.key)
|
||||
}()
|
||||
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || len(v.Toolchains) != 0 {
|
||||
t.Fatalf("build = %+v, want done with the archive and no toolchain named", v)
|
||||
}
|
||||
if n := len(r.bm.callsOf("build_go_build")); n != 2 {
|
||||
t.Errorf("%d go builds, want one on each of two nodes", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *testing.T) { // review round 4, 1
|
||||
r := newRig(t, archiveA)
|
||||
r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) // the nodes disagree: a bundle would wait
|
||||
r.b.Health = &HaveSet{} // no build module is healthy anywhere
|
||||
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") {
|
||||
t.Errorf("build = %+v, want failed: no node's build module is healthy", v)
|
||||
}
|
||||
have := &HaveSet{}
|
||||
have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
||||
r.b.Health = have
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
v := valueOf(t, r.st, r.key)
|
||||
if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix) {
|
||||
t.Errorf("build = %+v, want done on novox, the one node that qualifies, marked as one node", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1
|
||||
p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}})
|
||||
if err != nil || !p.NeedsToolchain(GoBuildModule) {
|
||||
t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(GoBuildModule), err)
|
||||
}
|
||||
p, _ = PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{archiveA}}})
|
||||
if p.NeedsToolchain(GoBuildModule) {
|
||||
t.Error("a plan of an archive needs a toolchain")
|
||||
}
|
||||
}
|
||||
|
||||
// 2. The digests-differ failure opens with its prefix, and nothing builds it again.
|
||||
func TestDifferingDigestsFailTheKeyWithThePrefixAndNothingBuildsItAgain(t *testing.T) { // review round 4, 2
|
||||
r := newRig(t, goBundle)
|
||||
r.bm.digests["ace/tool"] = "sha256:" + hex64('9')
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, "digests differ:") {
|
||||
t.Fatalf("build = %+v, want a failure opening with exactly %q", v, "digests differ:")
|
||||
}
|
||||
calls, writes := len(r.bm.calls), r.st.writes
|
||||
r.bm.digests = map[string]string{} // the nodes would now agree
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if len(r.bm.calls) != calls || r.st.writes != writes {
|
||||
t.Errorf("a failed-for-digests key was built again: %d calls (was %d), %d writes (was %d)", len(r.bm.calls), calls, r.st.writes, writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review round 4, 2
|
||||
st := newMem()
|
||||
seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"})
|
||||
writes := st.writes
|
||||
got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{TakeOver: true})
|
||||
if err != nil || got != nil || st.writes != writes {
|
||||
t.Errorf("claim = %v, %v, %d writes (was %d), want none and no write", got, err, st.writes, writes)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. A done key built on one node is built again on a second node once one qualifies.
|
||||
func TestADoneKeyOfOneNodeIsBuiltAgainOnASecondNodeAndTheDigestsCompared(t *testing.T) { // review round 4, 3
|
||||
r := newRig(t, goBundle)
|
||||
r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") {
|
||||
t.Fatalf("build = %+v, want done on one node, marked", v)
|
||||
}
|
||||
if got := OneNodeModules(v); len(got) != 1 || got[0] != GoBuildModule {
|
||||
t.Errorf("OneNodeModules = %v", got)
|
||||
}
|
||||
rev, calls := r.st.revs[r.key], len(r.bm.callsOf("build_go_build"))
|
||||
// Nothing changes while one node qualifies: looking again builds nothing.
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_go_build")) != calls {
|
||||
t.Fatal("a one-node key was built again with no second node")
|
||||
}
|
||||
// ace comes online: the key is built on ace and novox, as a new revision.
|
||||
r.pres.Put("ace", presenceValue(t, true), false)
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
v = valueOf(t, r.st, r.key)
|
||||
if v.State != delivery.BuildDone || strings.HasPrefix(v.Step, OneNodePrefix) || !strings.Contains(v.Step, "ace and novox") {
|
||||
t.Errorf("build = %+v, want done on ace and novox", v)
|
||||
}
|
||||
if r.st.revs[r.key] <= rev {
|
||||
t.Errorf("revision %d, was %d: not a new revision", r.st.revs[r.key], rev)
|
||||
}
|
||||
builds := r.bm.callsOf("build_go_build")
|
||||
if len(builds) != calls+2 || builds[len(builds)-2].node == builds[len(builds)-1].node {
|
||||
t.Errorf("%d builds (was %d), want one on each of two nodes", len(builds), calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondNodeThatDisagreesFailsTheDoneKeyWithTheDigestsPrefix(t *testing.T) { // review round 4, 3
|
||||
r := newRig(t, goBundle)
|
||||
r.pres.Put("ace", presenceValue(t, false), false)
|
||||
r.consider(t, "modules/m")
|
||||
r.bm.digests["ace/tool"] = "sha256:" + hex64('9')
|
||||
r.pres.Put("ace", presenceValue(t, true), false)
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, DigestsDifferPrefix) {
|
||||
t.Errorf("build = %+v, want failed with digests differ", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASecondBuildThatFailsForAnotherReasonLeavesTheDoneKeyStanding(t *testing.T) { // review round 4, 3
|
||||
r := newRig(t, goBundle)
|
||||
r.pres.Put("ace", presenceValue(t, false), false)
|
||||
first := r.consider(t, "modules/m")
|
||||
r.bm.fails["ace/build_go_fetch"] = "ace is out of disk"
|
||||
r.pres.Put("ace", presenceValue(t, true), false)
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); !delivery.SameBuilderBuild(v, first) {
|
||||
t.Errorf("build = %+v, want the first done build to stand: %+v", v, first)
|
||||
}
|
||||
}
|
||||
|
||||
// 4. A step never waits on a full node while another qualifying node has room.
|
||||
func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4
|
||||
p := &Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}}
|
||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks")
|
||||
total := 2 + 1 + 1 + 3
|
||||
var waiting int
|
||||
for i := 1; i <= 4; i++ { // four modules choose before any lane has started
|
||||
room := 0
|
||||
for _, n := range []string{"ace", "g14", "novox", "shanks"} {
|
||||
room += p.CapacityOf(n) - p.Running()[n] - p.Reserved()[n]
|
||||
}
|
||||
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
|
||||
if err != nil || len(nodes) != 2 {
|
||||
t.Fatalf("choice %d: %v, %v", i, nodes, err)
|
||||
}
|
||||
if room >= 2 {
|
||||
for _, n := range nodes {
|
||||
if p.Reserved()[n] > p.CapacityOf(n) {
|
||||
t.Errorf("choice %d: %v put a lane on full %s while %d places were free", i, nodes, n, room)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
waiting += 2 - room
|
||||
}
|
||||
}
|
||||
if want := 4*2 - total; waiting != want {
|
||||
t.Errorf("%d lanes wait, want %d: the 8 lanes less the %d places", waiting, want, total)
|
||||
}
|
||||
loads := p.Reserved()
|
||||
for n, c := range loads {
|
||||
if c > p.CapacityOf(n) && n != "shanks" && n != "ace" && n != "g14" && n != "novox" {
|
||||
t.Errorf("unknown node %s", n)
|
||||
}
|
||||
}
|
||||
if loads["shanks"] != 3 || loads["ace"] != 2 || loads["g14"]+loads["novox"] != 3 {
|
||||
t.Errorf("reservations = %v, want shanks 3, ace 2 and the single-place nodes taking the rest", loads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *testing.T) { // review round 4, 4
|
||||
p := &Places{}
|
||||
nodes, _, _ := ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
|
||||
if len(nodes) != 1 || p.Reserved()["ace"] != 1 {
|
||||
t.Fatalf("nodes %v, reserved %v", nodes, p.Reserved())
|
||||
}
|
||||
release, err := p.Acquire(context.Background(), "ace")
|
||||
if err != nil || p.Reserved()["ace"] != 0 || p.Running()["ace"] != 1 {
|
||||
t.Fatalf("acquire: %v, reserved %v, running %v", err, p.Reserved(), p.Running())
|
||||
}
|
||||
release()
|
||||
ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
|
||||
p.Cancel("ace")
|
||||
if len(p.Reserved()) != 0 {
|
||||
t.Errorf("reserved after Cancel = %v", p.Reserved())
|
||||
}
|
||||
}
|
||||
|
||||
// 5. A node whose build module is not healthy is not chosen.
|
||||
func TestHaveSetKnowsOnlyBuildModulesRunningAndHealthy(t *testing.T) { // review round 4, 5
|
||||
var s HaveSet
|
||||
if s.Healthy("ace", GoBuildModule) {
|
||||
t.Error("a module with no have is healthy")
|
||||
}
|
||||
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) || !s.Healthy("ace", GoBuildModule) {
|
||||
t.Error("a running, healthy module is not")
|
||||
}
|
||||
if s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) {
|
||||
t.Error("the same state counted as a change")
|
||||
}
|
||||
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) || s.Healthy("ace", GoBuildModule) {
|
||||
t.Error("a module whose health failed is healthy")
|
||||
}
|
||||
s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
||||
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseFailed, delivery.HealthOK), false) || s.Healthy("ace", GoBuildModule) {
|
||||
t.Error("a module whose phase failed is healthy")
|
||||
}
|
||||
s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
||||
if !s.Put("ace.mesh-build-go", nil, true) || s.Healthy("ace", GoBuildModule) {
|
||||
t.Error("a deleted have is healthy")
|
||||
}
|
||||
if s.Put("ace.node", []byte(`{}`), false) || s.Put("ace.mesh-build-go", []byte(`not json`), false) {
|
||||
t.Error("a node's key or a non-have counted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review round 4, 5
|
||||
r := newRig(t, goBundle)
|
||||
have := &HaveSet{}
|
||||
have.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) // go is not the recipe's
|
||||
have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
||||
r.b.Health = have
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on one node, novox") {
|
||||
t.Fatalf("build = %+v, want done on novox alone", v)
|
||||
}
|
||||
for _, c := range r.bm.calls {
|
||||
if c.node == "ace" {
|
||||
t.Errorf("a step was called on ace, whose build module is unhealthy: %+v", c)
|
||||
}
|
||||
}
|
||||
have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false)
|
||||
r.st.vals = map[string][]byte{} // a fresh key
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") {
|
||||
t.Errorf("build = %+v, want failed: no node has a healthy build module", v)
|
||||
}
|
||||
}
|
||||
|
||||
// 6. A key found building is taken over, whoever holds it.
|
||||
func TestABuildLeftBuildingByAnyBuilderIsTakenOverAtStart(t *testing.T) { // review round 4, 6; 1607/AC3, ADR 0478 clauses 5, 6
|
||||
st := newMem()
|
||||
seed(t, st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.a"})
|
||||
if got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); err != nil || got == nil {
|
||||
t.Errorf("own building key: claim = %v, %v, want it taken again", got, err)
|
||||
}
|
||||
if got, err := (Claimer{Store: st, By: "ace.b"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{TakeOver: true}); err != nil || got == nil || got.Build.By != "novox.a" {
|
||||
t.Errorf("another node's building key at start: claim = %v, %v, want it taken over", got, err)
|
||||
}
|
||||
if got, _ := (Claimer{Store: st, By: "ace.b"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{}); got != nil {
|
||||
t.Errorf("another builder's building key after the start was taken: a running builder's key is its own")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuilderFindingAKeyBuildingByAnotherNodeBuildsItOnStart(t *testing.T) { // review round 4, 6
|
||||
r := newRig(t, goBundle)
|
||||
seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace.builder", Manifest: manifestD})
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone || v.By != "novox.builder" {
|
||||
t.Errorf("build = %+v, want it taken over and done by novox.builder", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,519 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// ToolchainEntries are the nodes' toolchain entries of a language (ToolchainSet).
|
||||
type ToolchainEntries interface {
|
||||
Entries(language string) []delivery.NodeToolchain
|
||||
}
|
||||
|
||||
// toolPrefix is the start of each build module's tool names: <prefix>fetch and <prefix>build.
|
||||
var toolPrefix = map[string]string{GoBuildModule: "build_go_", NodeBuildModule: "build_node_"}
|
||||
|
||||
// Builder builds, dark, every new-form fingerprint the indexer writes (novox/hq ADR 0453 clause 4, ADR 0478): it is
|
||||
// handed each modules value as the watch of the indexer's bucket sees it, and for each module with a fingerprint and an
|
||||
// artifact to build claims the fingerprint's build key, builds it through the build modules on the nodes
|
||||
// the toolchains choose, and writes the key done or failed. It holds no timer; a build starts from a modules value or a
|
||||
// change of the toolchains.
|
||||
type Builder struct {
|
||||
Claimer Claimer
|
||||
Runner Runner
|
||||
Toolchains ToolchainEntries
|
||||
// Presence says which nodes are online (the watchman's); a node that is not is not chosen.
|
||||
Presence interface{ Online(node string) bool }
|
||||
Manifests Manifests
|
||||
// Health says whether a node's build module is running and healthy (the node-engine's have); a node whose build module
|
||||
// is not is not chosen. Nil means every node's is.
|
||||
Health interface {
|
||||
Healthy(node, module string) bool
|
||||
}
|
||||
// Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node.
|
||||
Places *Places
|
||||
// Nodes is how many build nodes build each fingerprint: two until release R4e (ADR 0478 clause 4).
|
||||
Nodes int
|
||||
// Limit is how many builds run at once; four when 0.
|
||||
Limit int
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
once sync.Once
|
||||
sem chan struct{}
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
known map[string]delivery.Module // by modules key
|
||||
inflight map[string]bool // by build key
|
||||
again map[string]delivery.Module // by build key: the newest value that arrived while the key was being built
|
||||
plans map[string]planned // by manifest digest
|
||||
tried map[string]bool // by build key: looked at since this process started
|
||||
// epoch counts the changes of the nodes (online, healthy, a toolchain) seen since this process started; a done key
|
||||
// whose second-node build failed is held back until it moves (issue 1664).
|
||||
epoch uint64
|
||||
secondAt map[string]uint64 // by build key: the epoch a second-node build of a one-node key failed at
|
||||
}
|
||||
|
||||
type planned struct {
|
||||
plan Plan
|
||||
err error
|
||||
}
|
||||
|
||||
func (b *Builder) init() {
|
||||
b.once.Do(func() {
|
||||
limit := b.Limit
|
||||
if limit <= 0 {
|
||||
limit = 4
|
||||
}
|
||||
b.sem = make(chan struct{}, limit)
|
||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||
b.secondAt = map[string]uint64{}
|
||||
if b.Places == nil {
|
||||
b.Places = &Places{}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Builder) logf(format string, args ...any) {
|
||||
if b.Logf != nil {
|
||||
b.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait returns when every build started so far has ended.
|
||||
func (b *Builder) Wait() { b.wg.Wait() }
|
||||
|
||||
// Module takes one value of the indexer's modules bucket in (the key is the module's name) and builds it if it is
|
||||
// to be built. A value that is not a module is said and left.
|
||||
func (b *Builder) Module(ctx context.Context, key string, value []byte) {
|
||||
b.init()
|
||||
m, err := delivery.DecodeModule(value)
|
||||
if err != nil {
|
||||
b.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
b.known[key] = m
|
||||
b.mu.Unlock()
|
||||
b.consider(ctx, key, m)
|
||||
}
|
||||
|
||||
// NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or
|
||||
// stopped being, or a toolchain moved. A done key built on one node whose second-node build failed is tried again only
|
||||
// after such a change, not at every modules value (issue 1664).
|
||||
func (b *Builder) NodesChanged(ctx context.Context) {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
b.epoch++
|
||||
b.mu.Unlock()
|
||||
b.Reconsider(ctx)
|
||||
}
|
||||
|
||||
// Sweep closes, at start, every key left building for a fingerprint its module no longer has (issue 1660, ADR 0478
|
||||
// clause 6): the key can never be built, since nothing asks for it, and would stay building for ever. It fails the key
|
||||
// naming who left it. A key of the fingerprint the module has now is not touched: the build takes it over. Call it
|
||||
// once the modules bucket has been read, so every module is known.
|
||||
func (b *Builder) Sweep(ctx context.Context) error {
|
||||
b.init()
|
||||
keys, err := b.Claimer.Store.Keys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.mu.Lock()
|
||||
current := map[string]bool{}
|
||||
for name, m := range b.known {
|
||||
if m.FP2 != "" {
|
||||
if k, err := delivery.BuilderBuildKey(name, m.FP2); err == nil {
|
||||
current[k] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
for _, key := range keys {
|
||||
if current[key] {
|
||||
continue
|
||||
}
|
||||
value, rev, found, err := b.Claimer.Store.Get(ctx, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
cur, err := delivery.DecodeBuilderBuild(value)
|
||||
if err != nil || cur.State != delivery.BuildBuilding {
|
||||
continue
|
||||
}
|
||||
closed := delivery.BuilderBuild{State: delivery.BuildFailed, By: b.Claimer.By, Manifest: cur.Manifest, Step: cur.Step,
|
||||
Error: fmt.Sprintf("closed at start: left building by %s, and the module no longer has this fingerprint, so nothing will build it", cur.By)}
|
||||
data, err := delivery.EncodeBuilderBuild(closed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := b.Claimer.Store.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
|
||||
return fmt.Errorf("%s: %w", key, err)
|
||||
}
|
||||
b.logf("%s: closed, left building by %s for a fingerprint the module no longer has", key, cur.By)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Forget takes a module out (its key was deleted).
|
||||
func (b *Builder) Forget(key string) {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
delete(b.known, key)
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
// Reconsider looks at every module again: the toolchains changed, so a build that failed for them may build now.
|
||||
func (b *Builder) Reconsider(ctx context.Context) {
|
||||
b.init()
|
||||
b.mu.Lock()
|
||||
known := make(map[string]delivery.Module, len(b.known))
|
||||
for k, m := range b.known {
|
||||
known[k] = m
|
||||
}
|
||||
b.mu.Unlock()
|
||||
for _, k := range sortedKeys(known) {
|
||||
b.consider(ctx, k, known[k])
|
||||
}
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]delivery.Module) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// planOf is the plan of the manifest with this digest, read once: a manifest by digest never changes. A manifest that
|
||||
// cannot be read is not kept, so it is read again when the module is.
|
||||
func (b *Builder) planOf(ctx context.Context, digest string) (Plan, error, bool) {
|
||||
b.mu.Lock()
|
||||
p, ok := b.plans[digest]
|
||||
b.mu.Unlock()
|
||||
if ok {
|
||||
return p.plan, p.err, true
|
||||
}
|
||||
m, err := b.Manifests.Manifest(ctx, digest)
|
||||
if err != nil {
|
||||
return Plan{}, fmt.Errorf("its manifest %s cannot be read: %w", digest, err), false
|
||||
}
|
||||
plan, err := PlanOf(m)
|
||||
b.mu.Lock()
|
||||
b.plans[digest] = planned{plan, err}
|
||||
b.mu.Unlock()
|
||||
return plan, err, true
|
||||
}
|
||||
|
||||
func (b *Builder) consider(ctx context.Context, module string, m delivery.Module) {
|
||||
if m.FP2 == "" || m.Manifest == "" {
|
||||
return
|
||||
}
|
||||
key, err := delivery.BuilderBuildKey(module, m.FP2)
|
||||
if err != nil {
|
||||
b.logf("modules.%s: %v", module, err)
|
||||
return
|
||||
}
|
||||
b.mu.Lock()
|
||||
if b.inflight[key] {
|
||||
b.again[key] = m // looked at once more when the build ends, so a change is never lost
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.inflight[key] = true
|
||||
b.mu.Unlock()
|
||||
b.wg.Add(1)
|
||||
go func() {
|
||||
defer b.wg.Done()
|
||||
b.sem <- struct{}{}
|
||||
defer func() { <-b.sem }()
|
||||
for {
|
||||
b.try(ctx, module, m)
|
||||
b.mu.Lock()
|
||||
next, more := b.again[key]
|
||||
delete(b.again, key)
|
||||
if !more || ctx.Err() != nil {
|
||||
delete(b.inflight, key)
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.mu.Unlock()
|
||||
m = next
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// try claims the fingerprint and builds it, writing the key done or failed. A build that ends because ctx did is left
|
||||
// building by this builder: it takes it again when it starts (ADR 0478 clause 5).
|
||||
func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
||||
plan, planErr, read := b.planOf(ctx, m.Manifest)
|
||||
if read && planErr == nil && len(plan.Tasks) == 0 {
|
||||
return // nothing in the manifest for a builder to build
|
||||
}
|
||||
if planErr == nil {
|
||||
if why := b.stale(m, plan); why != "" {
|
||||
// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
|
||||
// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
|
||||
b.logf("%s at %s is not built: %s", module, m.FP2, why)
|
||||
return
|
||||
}
|
||||
}
|
||||
key, _ := delivery.BuilderBuildKey(module, m.FP2)
|
||||
b.mu.Lock()
|
||||
first := !b.tried[key]
|
||||
b.tried[key] = true
|
||||
epoch := b.epoch
|
||||
failedAt, failedBefore := b.secondAt[key]
|
||||
b.mu.Unlock()
|
||||
moreNodes := func(modules []string) bool {
|
||||
// A second build that failed is not tried again until the nodes change (issue 1664).
|
||||
return (!failedBefore || epoch > failedAt) && b.moreNodes(modules)
|
||||
}
|
||||
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, MoreNodes: moreNodes})
|
||||
if err != nil {
|
||||
b.logf("%s: %v", module, err)
|
||||
return
|
||||
}
|
||||
if cl == nil {
|
||||
return
|
||||
}
|
||||
result := b.build(ctx, module, m, plan, planErr)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if result.State == "" {
|
||||
b.logf("%s: left building, the toolchains moved under it", cl.Key)
|
||||
return
|
||||
}
|
||||
result.By, result.Manifest = b.Claimer.By, m.Manifest
|
||||
if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, DigestsDifferPrefix) {
|
||||
// A done key built again on a second node keeps its done value when the new build fails for another reason than
|
||||
// the digests: the first build stands, and a later change of the nodes looks again.
|
||||
b.logf("%s: built again on a second node and failed, the done build stands: %s", cl.Key, result.Error)
|
||||
b.mu.Lock()
|
||||
b.secondAt[cl.Key] = epoch
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if err := b.Claimer.Finish(ctx, cl, result); err != nil {
|
||||
b.logf("%s: the build could not be written: %v", cl.Key, err)
|
||||
return
|
||||
}
|
||||
if result.State == delivery.BuildFailed {
|
||||
b.logf("%s: failed: %s", cl.Key, result.Error)
|
||||
}
|
||||
}
|
||||
|
||||
// stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan
|
||||
// has agreed (novox/hq ADR 0465) must be the one the fingerprint was computed with (Module.Toolchains).
|
||||
func (b *Builder) stale(m delivery.Module, plan Plan) string {
|
||||
for _, bm := range plan.Modules() {
|
||||
if !plan.NeedsToolchain(bm) {
|
||||
continue // an archive is packed with no toolchain
|
||||
}
|
||||
language := LanguageOf[bm]
|
||||
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||
if none != "" {
|
||||
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
|
||||
}
|
||||
want, ok := m.Toolchains[language]
|
||||
if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func failed(format string, args ...any) delivery.BuilderBuild {
|
||||
return delivery.BuilderBuild{State: delivery.BuildFailed, Error: fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
// lane is the build of a plan's tasks of one build module on one node.
|
||||
type lane struct {
|
||||
module, node string
|
||||
tasks []Task
|
||||
digests map[string]string // artifact -> digest
|
||||
err error
|
||||
}
|
||||
|
||||
func (b *Builder) build(ctx context.Context, module string, m delivery.Module, plan Plan, planErr error) delivery.BuilderBuild {
|
||||
switch {
|
||||
case planErr != nil:
|
||||
return failed("%s", planErr)
|
||||
case m.Path == "":
|
||||
// Empty is not known (a root module's path is "."): the indexer writes it again (issue 1615).
|
||||
return failed("the module's key names no directory (path), so the build tools cannot be told where %s lives", module)
|
||||
}
|
||||
repository := delivery.RepositoryName(m.Repository)
|
||||
var lanes []*lane
|
||||
used := map[string]delivery.ToolchainUsed{}
|
||||
var steps, oneNode []string
|
||||
// Every node chosen holds a reservation until its lane takes the place; a build that ends before its lanes start gives them up.
|
||||
giveUp := func(nodes ...string) {
|
||||
for _, n := range nodes {
|
||||
b.Places.Cancel(n)
|
||||
}
|
||||
}
|
||||
giveUpLanes := func() {
|
||||
for _, l := range lanes {
|
||||
b.Places.Cancel(l.node)
|
||||
}
|
||||
}
|
||||
for _, bm := range plan.Modules() {
|
||||
language := LanguageOf[bm]
|
||||
entries := b.Toolchains.Entries(language)
|
||||
qualifies := func(node string) bool { return b.qualifies(node, bm) }
|
||||
var nodes []string
|
||||
if plan.NeedsToolchain(bm) {
|
||||
ns, tc, err := ChooseNodes(language, entries, b.Places, qualifies, b.Nodes)
|
||||
if err != nil {
|
||||
giveUpLanes()
|
||||
return failed("%s", err)
|
||||
}
|
||||
if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||
giveUp(ns...)
|
||||
giveUpLanes()
|
||||
return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
|
||||
}
|
||||
used[language] = delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
|
||||
nodes = ns
|
||||
} else {
|
||||
// An archive is packed with no toolchain (the fingerprint names none): any node of the build module that qualifies.
|
||||
ns, err := ChooseAnyNodes(language, entries, b.Places, qualifies, b.Nodes)
|
||||
if err != nil {
|
||||
giveUpLanes()
|
||||
return failed("%s", err)
|
||||
}
|
||||
nodes = ns
|
||||
}
|
||||
if len(nodes) == 1 {
|
||||
oneNode = append(oneNode, bm)
|
||||
steps = append(steps, fmt.Sprintf("%s on one node, %s", bm, nodes[0]))
|
||||
} else {
|
||||
sorted := append([]string(nil), nodes...)
|
||||
sort.Strings(sorted)
|
||||
steps = append(steps, fmt.Sprintf("%s on %s", bm, strings.Join(sorted, " and ")))
|
||||
}
|
||||
var tasks []Task
|
||||
for _, t := range plan.Tasks {
|
||||
if t.Module == bm {
|
||||
tasks = append(tasks, t)
|
||||
}
|
||||
}
|
||||
for _, n := range nodes {
|
||||
lanes = append(lanes, &lane{module: bm, node: n, tasks: tasks, digests: map[string]string{}})
|
||||
}
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
for _, l := range lanes {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); b.runLane(ctx, l, repository, m, module) }()
|
||||
}
|
||||
wg.Wait()
|
||||
var errs []string
|
||||
for _, l := range lanes {
|
||||
if l.err != nil {
|
||||
errs = append(errs, l.err.Error())
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
return failed("%s", strings.Join(errs, "; "))
|
||||
}
|
||||
var artifacts []delivery.BuiltArtifact
|
||||
for _, t := range plan.Tasks {
|
||||
byNode := map[string]string{}
|
||||
for _, l := range lanes {
|
||||
if d, ok := l.digests[t.Artifact]; ok && l.module == t.Module {
|
||||
byNode[l.node] = d
|
||||
}
|
||||
}
|
||||
d, err := Agree(t.Artifact, byNode)
|
||||
if err != nil {
|
||||
return failed("%s", err)
|
||||
}
|
||||
artifacts = append(artifacts, delivery.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d})
|
||||
}
|
||||
step := "built " + strings.Join(steps, "; ")
|
||||
if len(oneNode) > 0 {
|
||||
// Marked so that Claim builds the key again once a second node qualifies (ADR 0478 clause 4).
|
||||
step = OneNodePrefix + strings.Join(oneNode, ", ") + "): " + step
|
||||
}
|
||||
if len(used) == 0 {
|
||||
used = nil
|
||||
}
|
||||
return delivery.BuilderBuild{State: delivery.BuildDone, Step: step, Artifacts: artifacts, Toolchains: used}
|
||||
}
|
||||
|
||||
func (b *Builder) online(node string) bool { return b.Presence == nil || b.Presence.Online(node) }
|
||||
|
||||
// qualifies is whether the node can build with the build module now: it is online and the build module on it is healthy.
|
||||
func (b *Builder) qualifies(node, bm string) bool {
|
||||
return b.Places.CapacityOf(node) > 0 && b.online(node) && (b.Health == nil || b.Health.Healthy(node, bm))
|
||||
}
|
||||
|
||||
// moreNodes is whether any of these build modules has two nodes that qualify now.
|
||||
func (b *Builder) moreNodes(modules []string) bool {
|
||||
for _, bm := range modules {
|
||||
seen := map[string]bool{}
|
||||
n := 0
|
||||
for _, e := range b.Toolchains.Entries(LanguageOf[bm]) {
|
||||
if !seen[e.Node] && b.qualifies(e.Node, bm) {
|
||||
seen[e.Node] = true
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n >= 2 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// step runs one step of a build module; the lane that runs it holds its node's place.
|
||||
func (b *Builder) step(ctx context.Context, bm, node, tool string, args map[string]any) (json.RawMessage, error) {
|
||||
return b.Runner.Run(ctx, bm, node, toolPrefix[bm]+tool, args)
|
||||
}
|
||||
|
||||
// runLane takes a place on the node, fetches the commit there and builds each of the lane's artifacts, one step at a time.
|
||||
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string) {
|
||||
release, err := b.Places.Acquire(ctx, l.node)
|
||||
if err != nil {
|
||||
l.err = err
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
raw, err := b.step(ctx, l.module, l.node, "fetch", map[string]any{"repository": repository, "commit": m.Commit})
|
||||
if err != nil {
|
||||
l.err = err
|
||||
return
|
||||
}
|
||||
var fetched struct {
|
||||
Workspace string `json:"workspace"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" {
|
||||
l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw)
|
||||
return
|
||||
}
|
||||
for _, t := range l.tasks {
|
||||
raw, err := b.step(ctx, l.module, l.node, "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact})
|
||||
if err != nil {
|
||||
l.err = err
|
||||
return
|
||||
}
|
||||
var built struct {
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" {
|
||||
l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, l.node, raw)
|
||||
return
|
||||
}
|
||||
l.digests[t.Artifact] = built.Digest
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the
|
||||
// step's end into the steps (as mesh-build-go and mesh-build-node do), with a digest by node and artifact, or a failure.
|
||||
type buildModules struct {
|
||||
mu sync.Mutex
|
||||
n int
|
||||
calls []call
|
||||
steps *fakeSteps
|
||||
digests map[string]string // `<node>/<artifact>` -> digest; the default is `digest`
|
||||
fails map[string]string // `<node>/<tool>` -> the reason the step fails with
|
||||
// hold, when set before the builds start, blocks every call until it is closed; entered is sent to as a call arrives.
|
||||
hold chan struct{}
|
||||
entered chan struct{}
|
||||
}
|
||||
|
||||
func newBuildModules() *buildModules {
|
||||
return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}}
|
||||
}
|
||||
|
||||
func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
if b.hold != nil {
|
||||
select {
|
||||
case b.entered <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
<-b.hold
|
||||
}
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.calls = append(b.calls, call{node, key, body})
|
||||
b.n++
|
||||
id := fmt.Sprintf("s%d", b.n)
|
||||
module, tool, _ := strings.Cut(key, ".")
|
||||
args, _ := body.(map[string]any)
|
||||
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepPassed}
|
||||
if reason, ok := b.fails[node+"/"+tool]; ok {
|
||||
step.State, step.Reason = delivery.StepFailed, reason
|
||||
} else if strings.HasSuffix(tool, "_fetch") {
|
||||
step.Result = json.RawMessage(`{"workspace":"w-` + node + `"}`)
|
||||
} else {
|
||||
d := digest
|
||||
if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok {
|
||||
d = v
|
||||
}
|
||||
step.Result = json.RawMessage(`{"digest":"` + d + `"}`)
|
||||
}
|
||||
sk, _ := delivery.StepKey(node, id)
|
||||
b.steps.mu.Lock()
|
||||
b.steps.steps[module+"/"+sk] = step
|
||||
b.steps.mu.Unlock()
|
||||
return json.RawMessage(`{"step":"` + id + `","key":"` + sk + `","state":"running"}`), nil
|
||||
}
|
||||
|
||||
func (b *buildModules) callsOf(tool string) (out []call) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for _, c := range b.calls {
|
||||
if strings.HasSuffix(c.key, "."+tool) {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type fakeManifests struct {
|
||||
byDigest map[string]catalogue.Manifest
|
||||
}
|
||||
|
||||
func (f *fakeManifests) Manifest(_ context.Context, d string) (catalogue.Manifest, error) {
|
||||
m, ok := f.byDigest[d]
|
||||
if !ok {
|
||||
return catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", d)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func mdigest(c byte) string { return hex64(c) }
|
||||
|
||||
type rig struct {
|
||||
b *Builder
|
||||
st *memStore
|
||||
bm *buildModules
|
||||
tools *ToolchainSet
|
||||
pres *PresenceSet
|
||||
mf *fakeManifests
|
||||
key string
|
||||
}
|
||||
|
||||
func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
|
||||
t.Helper()
|
||||
r := &rig{st: newMem(), bm: newBuildModules(), tools: &ToolchainSet{}, pres: &PresenceSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
|
||||
for _, n := range []string{"ace", "novox"} {
|
||||
r.tools.Put(GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false)
|
||||
r.tools.Put(NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||
r.pres.Put(n, presenceValue(t, true), false)
|
||||
}
|
||||
r.b = &Builder{
|
||||
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second},
|
||||
Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf,
|
||||
}
|
||||
var err error
|
||||
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// agreedToolchains are what the rig's nodes report, and so what a fingerprint computed now names.
|
||||
var agreedToolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}
|
||||
|
||||
func moduleValue(t *testing.T, path string) []byte { return moduleValueWith(t, path, agreedToolchains) }
|
||||
|
||||
func moduleValueWith(t *testing.T, path string, toolchains map[string]delivery.ToolchainUsed) []byte {
|
||||
t.Helper()
|
||||
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp1, Manifest: mdigest('a'), Path: path, Toolchains: toolchains})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func (r *rig) consider(t *testing.T, path string) delivery.BuilderBuild {
|
||||
t.Helper()
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, path))
|
||||
r.b.Wait()
|
||||
if _, ok := r.st.vals[r.key]; !ok {
|
||||
t.Fatal("no build key was written")
|
||||
}
|
||||
return valueOf(t, r.st, r.key)
|
||||
}
|
||||
|
||||
var (
|
||||
goBundle = catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}
|
||||
tsBundle = catalogue.Artifact{Name: "web", Kind: "bundle", Language: "typescript"}
|
||||
archiveA = catalogue.Artifact{Name: "files", Kind: "archive"}
|
||||
imageArt = catalogue.Artifact{Name: "img", Kind: "image"}
|
||||
manifestD = mdigest('a')
|
||||
)
|
||||
|
||||
func TestABundleAndAnArchiveAreBuiltOnTwoNodesAndDoneWhenTheDigestsAgree(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4
|
||||
r := newRig(t, goBundle, archiveA, tsBundle)
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 {
|
||||
t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v)
|
||||
}
|
||||
for i, want := range []string{"files", "tool", "web"} {
|
||||
if a := v.Artifacts[i]; a.Name != want || a.Digest != digest || a.Reference != "" {
|
||||
t.Errorf("artifact %d = %+v, want %s with the digest and no reference (ADR 0478 clause 6)", i, a, want)
|
||||
}
|
||||
}
|
||||
if v.Artifacts[0].Kind != "archive" || v.Artifacts[1].Kind != "bundle" {
|
||||
t.Errorf("kinds = %+v", v.Artifacts)
|
||||
}
|
||||
if got := v.Toolchains["go"]; got.Version != "go1.27.1" || got.Recipe != "1" {
|
||||
t.Errorf("toolchains = %+v, want go and typescript named", v.Toolchains)
|
||||
}
|
||||
if got := v.Toolchains["typescript"]; got.Version != "tsc5" {
|
||||
t.Errorf("toolchains = %+v", v.Toolchains)
|
||||
}
|
||||
// Two fetches per build module, one on each node; every build names the module's directory.
|
||||
if n := len(r.bm.callsOf("build_go_fetch")); n != 2 {
|
||||
t.Errorf("%d go fetches, want one per node", n)
|
||||
}
|
||||
if n := len(r.bm.callsOf("build_go_build")); n != 4 { // tool and files, each on two nodes
|
||||
t.Errorf("%d go builds, want 4", n)
|
||||
}
|
||||
if n := len(r.bm.callsOf("build_node_build")); n != 2 {
|
||||
t.Errorf("%d node builds, want 2", n)
|
||||
}
|
||||
c := r.bm.callsOf("build_go_build")[0]
|
||||
if args := c.body.(map[string]any); args["module"] != "modules/m" || !strings.HasPrefix(fmt.Sprint(args["workspace"]), "w-") {
|
||||
t.Errorf("build args = %+v, want the module's path and the fetch's workspace", args)
|
||||
}
|
||||
f := r.bm.callsOf("build_go_fetch")[0].body.(map[string]any)
|
||||
if f["repository"] != "novox/mesh-catalog" || f["commit"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" {
|
||||
t.Errorf("fetch args = %+v", f)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDifferingDigestsFailTheBuildNamingBothNodes(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
|
||||
r := newRig(t, goBundle)
|
||||
r.bm.digests["ace/tool"] = "sha256:" + hex64('9')
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "ace") || !strings.Contains(v.Error, "novox") || !strings.Contains(v.Error, digest) || !strings.Contains(v.Error, hex64('9')) {
|
||||
t.Errorf("build = %+v, want failed naming both nodes and both digests", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOneQualifyingNodeBuildsOnceAndTheKeySaysSo(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
|
||||
r := newRig(t, goBundle)
|
||||
r.tools.Put(GoBuildModule, "ace.go", nil, true)
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "one node") || !strings.Contains(v.Step, "novox") {
|
||||
t.Errorf("build = %+v, want done with its step saying one node, novox", v)
|
||||
}
|
||||
if n := len(r.bm.callsOf("build_go_build")); n != 1 {
|
||||
t.Errorf("%d builds, want one", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailingStepFailsTheBuildWithItsReasonAndNodeAndIsTriedAgainAsANewRevision(t *testing.T) { // 1607/AC7, ADR 0429 clause 18
|
||||
r := newRig(t, goBundle)
|
||||
r.bm.fails["ace/build_go_build"] = "go build of tool: undefined: x"
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "ace") || !strings.Contains(v.Error, "undefined: x") {
|
||||
t.Fatalf("build = %+v, want failed with the node and the reason", v)
|
||||
}
|
||||
rev := r.st.revs[r.key]
|
||||
delete(r.bm.fails, "ace/build_go_build")
|
||||
v = r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev {
|
||||
t.Errorf("build = %+v at revision %d (was %d), want done as a new revision of the same key", v, r.st.revs[r.key], rev)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleWithAnImageGetsARefusedKeyNeverBuiltAgain(t *testing.T) { // 1607/AC2, ADR 0478 clause 2
|
||||
r := newRig(t, archiveA, imageArt)
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, RefusedPrefix) || !strings.Contains(v.Error, "image") || !strings.Contains(v.Error, "build-agent") {
|
||||
t.Fatalf("build = %+v, want failed 'refused kind: image' naming build-agent", v)
|
||||
}
|
||||
if len(r.bm.calls) != 0 {
|
||||
t.Errorf("%d calls, want none for a refused kind", len(r.bm.calls))
|
||||
}
|
||||
writes := r.st.writes
|
||||
r.consider(t, "modules/m")
|
||||
if r.st.writes != writes {
|
||||
t.Errorf("a refused key was written again (%d writes, was %d)", r.st.writes, writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleKeyWithNoPathFailsNamingThatReason(t *testing.T) { // 1607/AC8, ADR 0478 clause 8
|
||||
r := newRig(t, goBundle)
|
||||
v := r.consider(t, "")
|
||||
if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "path") || len(r.bm.calls) != 0 {
|
||||
t.Errorf("build = %+v with %d calls, want failed naming the missing path and nothing called", v, len(r.bm.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableManifestFailsTheBuild(t *testing.T) {
|
||||
r := newRig(t, goBundle)
|
||||
delete(r.mf.byDigest, mdigest('a'))
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "manifest") {
|
||||
t.Errorf("build = %+v, want failed naming the manifest", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleWithNoFingerprintOrNoArtifactIsNotBuilt(t *testing.T) { // 1607/AC2
|
||||
r := newRig(t)
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
data, _ := delivery.EncodeModule(delivery.Module{Repository: "novox/r", Commit: "c", FP: "src1:" + hex64('f'), None2: "toolchain not yet reported"})
|
||||
r.b.Module(context.Background(), "n", data)
|
||||
r.b.Wait()
|
||||
if len(r.st.vals) != 0 {
|
||||
t.Errorf("keys %v were written: a module with no new-form fingerprint or no artifact has nothing to build", r.st.vals)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t *testing.T) { // 1607/AC3, AC7
|
||||
r := newRig(t, goBundle)
|
||||
seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD})
|
||||
if v := r.consider(t, "modules/m"); v.State != delivery.BuildDone {
|
||||
t.Fatalf("build = %+v, want the builder's own building key built", v)
|
||||
}
|
||||
calls := len(r.bm.calls)
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if len(r.bm.calls) != calls {
|
||||
t.Errorf("a done key was built again (%d calls, was %d)", len(r.bm.calls), calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Two instances that start together on one key: the second finds the key building and takes it over (ADR 0478 clause 6:
|
||||
// a key found building at start is taken over, whoever holds it), so both may build; the key is written once, by the one
|
||||
// whose compare-and-set comes first, and the other's write is refused, never overwriting it.
|
||||
func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7, review round 4
|
||||
r := newRig(t, goBundle)
|
||||
other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf}
|
||||
value := moduleValue(t, "modules/m")
|
||||
var wg sync.WaitGroup
|
||||
for _, b := range []*Builder{r.b, other} {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); b.Module(context.Background(), "m", value); b.Wait() }()
|
||||
}
|
||||
wg.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(v.Artifacts) != 1 {
|
||||
t.Errorf("build = %+v, want one done key", v)
|
||||
}
|
||||
if n := len(r.bm.callsOf("build_go_build")); n < 2 || n%2 != 0 {
|
||||
t.Errorf("%d builds, want two for each instance that built", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModulesValueThatArrivesWhileItsKeyIsBeingBuiltIsLookedAtAgainWhenItEnds(t *testing.T) { // 1607/AC6 (no event is dropped, no timer)
|
||||
r := newRig(t, goBundle)
|
||||
r.bm.fails["ace/build_go_build"] = "go build of tool: boom"
|
||||
r.bm.hold, r.bm.entered = make(chan struct{}), make(chan struct{}, 8)
|
||||
value := moduleValue(t, "modules/m")
|
||||
r.b.Module(context.Background(), "m", value)
|
||||
<-r.bm.entered // the first build is under way
|
||||
r.b.Module(context.Background(), "m", value)
|
||||
r.b.Reconsider(context.Background())
|
||||
delete(r.bm.fails, "ace/build_go_build") // what the event is about: the build can succeed now
|
||||
close(r.bm.hold)
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
|
||||
t.Errorf("build = %+v, want the failed key built again as a new revision after the first build ended", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOtherwise(t *testing.T) { // 1615/AC4
|
||||
r := newRig(t, goBundle)
|
||||
old := map[string]delivery.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}}
|
||||
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old))
|
||||
r.b.Wait()
|
||||
if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
|
||||
t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls))
|
||||
}
|
||||
// A recipe that differs is a different toolchain too, and a value that names none (written before) is not built.
|
||||
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "0"}}))
|
||||
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
|
||||
r.b.Wait()
|
||||
if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
|
||||
t.Errorf("keys %v and %d calls for a recipe that differs or a value that names no toolchain", r.st.vals, len(r.bm.calls))
|
||||
}
|
||||
// The toolchain moves under a key that is not built yet: the old fingerprint stays unbuilt, the new one builds.
|
||||
r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false)
|
||||
r.tools.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false)
|
||||
r.b.Reconsider(context.Background())
|
||||
r.b.Wait()
|
||||
if len(r.st.vals) != 0 {
|
||||
t.Errorf("keys %v: the old fingerprint was built with the new toolchain", r.st.vals)
|
||||
}
|
||||
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}}))
|
||||
r.b.Wait()
|
||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Toolchains["go"].Version != "go1.27.2" {
|
||||
t.Errorf("build = %+v, want done with go1.27.2", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4
|
||||
r := newRig(t, goBundle)
|
||||
r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false)
|
||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||
r.b.Wait()
|
||||
if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
|
||||
t.Errorf("keys %v and %d calls while the nodes disagree", r.st.vals, len(r.bm.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/AC4
|
||||
r := newRig(t, goBundle)
|
||||
if v := r.consider(t, delivery.RootPath); v.State != delivery.BuildDone {
|
||||
t.Fatalf("build = %+v, want done for a root module", v)
|
||||
}
|
||||
if args := r.bm.callsOf("build_go_build")[0].body.(map[string]any); args["module"] != "." {
|
||||
t.Errorf("build args = %+v, want module \".\"", args)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatIsNotOnlineIsNotChosenAndComingBackWakesTheBuild(t *testing.T) { // 1607 review 3
|
||||
r := newRig(t, goBundle)
|
||||
r.tools.Put(GoBuildModule, "g14.go", tcValue(t, "go", "go1.27.1"), false) // a laptop that reports its toolchain and sleeps
|
||||
v := r.consider(t, "modules/m")
|
||||
if v.State != delivery.BuildDone {
|
||||
t.Fatalf("build = %+v", v)
|
||||
}
|
||||
for _, c := range r.bm.calls {
|
||||
if c.node == "g14" {
|
||||
t.Errorf("a call went to g14, which is not online: %+v", c)
|
||||
}
|
||||
}
|
||||
// With no node online the key fails naming why; a node coming online builds it as a new revision.
|
||||
r2 := newRig(t, goBundle)
|
||||
r2.pres.Put("ace", nil, true)
|
||||
r2.pres.Put("novox", nil, true)
|
||||
if v := r2.consider(t, "modules/m"); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "online") {
|
||||
t.Fatalf("build = %+v, want failed because no node is online", v)
|
||||
}
|
||||
r2.pres.Put("ace", presenceValue(t, true), false)
|
||||
r2.b.Reconsider(context.Background())
|
||||
r2.b.Wait()
|
||||
if v := valueOf(t, r2.st, r2.key); v.State != delivery.BuildDone {
|
||||
t.Errorf("build = %+v, want done once a node is online", v)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Caller calls a tool of a module on one named machine (mesh-sdk process.Conn.CallOn; ADR 0469 clause 3).
|
||||
type Caller interface {
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// Steps waits for the end of a build module's long step: the value of its key `<node>.<id>` in the module's steps state
|
||||
// (ADR 0469 clause 2) once its state is not running. It is a watch of that key, not a poll; it returns when the step
|
||||
// ends or ctx does.
|
||||
type Steps interface {
|
||||
Wait(ctx context.Context, module, key string) (delivery.Step, error)
|
||||
}
|
||||
|
||||
// Runner runs a build module's long steps: the call answers at once with the step's key, and the result is read from that
|
||||
// key when the step ends (ADR 0469 clause 2).
|
||||
type Runner struct {
|
||||
Caller Caller
|
||||
Steps Steps
|
||||
// Bound is how long a step may run before the builder stops waiting for it.
|
||||
Bound time.Duration
|
||||
}
|
||||
|
||||
// Run calls tool of the build module on node with args, waits for the step's end and answers its result. A step that
|
||||
// failed or was refused is an error naming the node, the state and the reason.
|
||||
func (r Runner) Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) {
|
||||
raw, err := r.Caller.CallOn(node, module+"."+tool, args)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
|
||||
}
|
||||
var answer struct {
|
||||
Step string `json:"step"`
|
||||
Key string `json:"key"`
|
||||
State string `json:"state"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &answer); err != nil {
|
||||
return nil, fmt.Errorf("%s on %s answered what is not a step: %w", tool, node, err)
|
||||
}
|
||||
if answer.Key == "" {
|
||||
return nil, fmt.Errorf("%s on %s answered no step key: %s", tool, node, raw)
|
||||
}
|
||||
if n, _, err := delivery.ParseStepKey(answer.Key); err != nil || n != node {
|
||||
return nil, fmt.Errorf("%s on %s answered the step key %q, which is not a key of that node", tool, node, answer.Key)
|
||||
}
|
||||
wait, cancel := context.WithTimeoutCause(ctx, r.Bound, fmt.Errorf("the step %s did not end within %s", answer.Key, r.Bound))
|
||||
defer cancel()
|
||||
step, err := r.Steps.Wait(wait, module, answer.Key)
|
||||
if err != nil {
|
||||
if cause := context.Cause(wait); cause != nil && ctx.Err() == nil {
|
||||
err = cause
|
||||
}
|
||||
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
|
||||
}
|
||||
if step.State != delivery.StepPassed {
|
||||
return nil, fmt.Errorf("%s on %s: the step %s %s: %s", tool, node, answer.Key, step.State, step.Reason)
|
||||
}
|
||||
return step.Result, nil
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
type call struct {
|
||||
node, key string
|
||||
body any
|
||||
}
|
||||
|
||||
// fakeCaller answers a long tool as the build modules do: at once, with the step and its key.
|
||||
type fakeCaller struct {
|
||||
calls []call
|
||||
answer string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
f.calls = append(f.calls, call{node, key, body})
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
return json.RawMessage(f.answer), nil
|
||||
}
|
||||
|
||||
// fakeSteps holds the steps values by `<module>/<key>`; Wait on a key it does not hold blocks until the context ends.
|
||||
type fakeSteps struct {
|
||||
mu sync.Mutex
|
||||
steps map[string]delivery.Step
|
||||
// during, when set, is called as a wait starts and the function it answers as the wait ends.
|
||||
during func(key string) func()
|
||||
}
|
||||
|
||||
func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
|
||||
f.mu.Lock()
|
||||
s, ok := f.steps[module+"/"+key]
|
||||
during := f.during
|
||||
f.mu.Unlock()
|
||||
if ok {
|
||||
if during != nil {
|
||||
defer during(key)()
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
<-ctx.Done()
|
||||
return delivery.Step{}, context.Cause(ctx)
|
||||
}
|
||||
|
||||
func stepAt(node, id, state, reason string, result string) delivery.Step {
|
||||
s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason}
|
||||
if result != "" {
|
||||
s.Result = json.RawMessage(result)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestRunCallsTheToolOnTheNodeAndReadsTheStepsKey(t *testing.T) { // 1607/AC5
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", delivery.StepPassed, "", `{"workspace":"w1","commit":"c"}`)}}
|
||||
res, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", map[string]any{"repository": "novox/r", "commit": "c"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(caller.calls) != 1 || caller.calls[0].node != "ace" || caller.calls[0].key != "mesh-build-go.build_go_fetch" {
|
||||
t.Errorf("calls = %+v, want one on ace for mesh-build-go.build_go_fetch", caller.calls)
|
||||
}
|
||||
var fr struct{ Workspace string }
|
||||
if err := json.Unmarshal(res, &fr); err != nil || fr.Workspace != "w1" {
|
||||
t.Errorf("result = %s (%v), want the step's result", res, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunFailsOnAFailedOrRefusedStepNamingItsReason(t *testing.T) { // 1607/AC5
|
||||
for _, state := range []string{delivery.StepFailed, delivery.StepRefused} {
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", state, "go is go1.26.0, the recipe builds with go1.27.1", "")}}
|
||||
_, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), state) || !strings.Contains(err.Error(), "ace") {
|
||||
t.Errorf("%s: error = %v, want the node, the state and the reason", state, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsWaitingAtTheBound(t *testing.T) { // 1607/AC5
|
||||
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
|
||||
_, err := Runner{Caller: caller, Steps: &fakeSteps{}, Bound: 20 * time.Millisecond}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
|
||||
if err == nil || !strings.Contains(err.Error(), "ace.abc1") || !strings.Contains(err.Error(), "did not end") {
|
||||
t.Errorf("error = %v, want the step key and that it did not end within its bound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode(t *testing.T) { // 1607/AC5
|
||||
for _, answer := range []string{`{"state":"running"}`, `{"step":"abc1","key":"novox.abc1","state":"running"}`, `not json`} {
|
||||
_, err := Runner{Caller: &fakeCaller{answer: answer}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
|
||||
if err == nil {
|
||||
t.Errorf("answer %s: no error", answer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunReturnsTheCallsOwnError(t *testing.T) { // 1607/AC5
|
||||
boom := errors.New("no responders")
|
||||
_, err := Runner{Caller: &fakeCaller{err: boom}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
|
||||
if !errors.Is(err, boom) || !strings.Contains(err.Error(), "ace") {
|
||||
t.Errorf("error = %v, want the call's error naming the node", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed).
|
||||
var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"}
|
||||
|
||||
// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches
|
||||
// put them in. Safe for use from the watches and from a build at once.
|
||||
type ToolchainSet struct {
|
||||
mu sync.Mutex
|
||||
by map[string]map[string]delivery.Toolchain // language -> node -> entry
|
||||
}
|
||||
|
||||
// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key <node>.<language>) or its
|
||||
// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going).
|
||||
// A key or a value that is not an entry of the module's language is left out.
|
||||
func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool {
|
||||
language, ok := LanguageOf[module]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
node, keyLanguage, err := delivery.ParseToolchainKey(key)
|
||||
if err != nil || keyLanguage != language {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if deleted {
|
||||
if _, had := s.by[language][node]; !had {
|
||||
return false
|
||||
}
|
||||
delete(s.by[language], node)
|
||||
return true
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err != nil || tc.Language != language {
|
||||
return false
|
||||
}
|
||||
if s.by == nil {
|
||||
s.by = map[string]map[string]delivery.Toolchain{}
|
||||
}
|
||||
if s.by[language] == nil {
|
||||
s.by[language] = map[string]delivery.Toolchain{}
|
||||
}
|
||||
cur, had := s.by[language][node]
|
||||
s.by[language][node] = tc
|
||||
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
|
||||
}
|
||||
|
||||
// Entries are the nodes' entries of a language, in no particular order.
|
||||
func (s *ToolchainSet) Entries(language string) []delivery.NodeToolchain {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []delivery.NodeToolchain
|
||||
for node, tc := range s.by[language] {
|
||||
out = append(out, delivery.NodeToolchain{Node: node, Toolchain: tc})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package build
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func tcValue(t *testing.T, language, version string) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6
|
||||
var s ToolchainSet
|
||||
if !s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
|
||||
t.Error("a first entry did not count as a change")
|
||||
}
|
||||
if s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
|
||||
t.Error("the same entry again counted as a change")
|
||||
}
|
||||
s.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.1"), false)
|
||||
s.Put(NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||
if got := s.Entries("go"); len(got) != 2 {
|
||||
t.Errorf("go entries = %+v, want two", got)
|
||||
}
|
||||
if got := s.Entries("typescript"); len(got) != 1 || got[0].Node != "ace" || got[0].Version != "tsc5" {
|
||||
t.Errorf("typescript entries = %+v", got)
|
||||
}
|
||||
if !s.Put(GoBuildModule, "ace.go", nil, true) || len(s.Entries("go")) != 1 {
|
||||
t.Error("a delete did not remove the node's entry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestToolchainSetLeavesOutWhatIsNotAToolchainOfTheModulesLanguage(t *testing.T) { // 1607/AC6
|
||||
var s ToolchainSet
|
||||
if s.Put(GoBuildModule, "novox.typescript", tcValue(t, "typescript", "x"), false) {
|
||||
t.Error("a typescript toolchain in the go build module's bucket counted")
|
||||
}
|
||||
if s.Put(GoBuildModule, "novox.go", []byte("not json"), false) || s.Put("other-module", "novox.go", tcValue(t, "go", "x"), false) {
|
||||
t.Error("garbage or an unknown module counted")
|
||||
}
|
||||
if len(s.Entries("go")) != 0 {
|
||||
t.Error("an entry was kept")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package manifest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
|
||||
// Invokes :590, Streams :613, Reads :659; StreamDeclaration, streams.go), an internal package of another repository.
|
||||
// These tests decode the manifest strictly into the fields those types name (DisallowUnknownFields) and hold the
|
||||
// rules of issue 1607 (ADR 0453 §4, ADR 0478). The merge gate's module check proves the controller takes the manifest.
|
||||
|
||||
type state struct {
|
||||
Name string `json:"name"`
|
||||
History int `json:"history,omitempty"`
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
Language string `json:"language"`
|
||||
System string `json:"system"`
|
||||
From string `json:"from"`
|
||||
Binary string `json:"binary"`
|
||||
}
|
||||
|
||||
type manifest struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
Slug string `json:"slug"`
|
||||
Consumes []string `json:"consumes"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Invokes []string `json:"invokes"`
|
||||
Config map[string]struct {
|
||||
Type string `json:"type"`
|
||||
Description string `json:"description"`
|
||||
Pattern string `json:"pattern"`
|
||||
Default string `json:"default"`
|
||||
} `json:"config"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
SecretsOwner string `json:"secrets-owner"`
|
||||
Resources []map[string]any `json:"resources"`
|
||||
Build struct {
|
||||
Artifacts []artifact `json:"artifacts"`
|
||||
} `json:"build"`
|
||||
}
|
||||
|
||||
func load(t *testing.T) (manifest, map[string]json.RawMessage) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile("module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m manifest
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&m); err != nil {
|
||||
t.Fatalf("the controller's reader refuses a key this manifest has: %v", err)
|
||||
}
|
||||
var top map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &top); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m, top
|
||||
}
|
||||
|
||||
func sorted(s []string) []string {
|
||||
s = slices.Clone(s)
|
||||
slices.Sort(s)
|
||||
return s
|
||||
}
|
||||
|
||||
func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
||||
m, top := load(t)
|
||||
if m.Module != "mesh-delivery-builder" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "builds", History: 5}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains",
|
||||
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
|
||||
t.Errorf("invokes = %v, want only the build modules' fetch and build", m.Invokes)
|
||||
}
|
||||
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts.
|
||||
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
t.Errorf("own-secrets = %v, want exactly broker", m.OwnSecrets)
|
||||
}
|
||||
if m.SecretsOwner != "mesh-delivery-builder" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the builder has none", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestManifestProcess(t *testing.T) { // 1607/AC1
|
||||
m, _ := load(t)
|
||||
var procs []map[string]any
|
||||
for _, r := range m.Resources {
|
||||
if r["type"] == "process" {
|
||||
procs = append(procs, r)
|
||||
}
|
||||
if r["type"] == "tool" || r["type"] == "seat" {
|
||||
t.Errorf("resource %v: the builder serves no tool and holds no seat", r)
|
||||
}
|
||||
}
|
||||
if len(procs) != 1 {
|
||||
t.Fatalf("%d process resources, want one", len(procs))
|
||||
}
|
||||
p := procs[0]
|
||||
if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-builder"}) {
|
||||
t.Errorf("process = %v", p)
|
||||
}
|
||||
env, _ := p["env"].(map[string]any)
|
||||
// The build capacity of each node is the module's setting `capacity` (issue 1661, ADR 0478 clause 3), laid into the
|
||||
// settings file the program reads; empty is one step at a time everywhere. The environment holds no capacity.
|
||||
if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_SETTINGS"] != "${dir:state}/settings.json" {
|
||||
t.Errorf("env = %v", p["env"])
|
||||
}
|
||||
if !reflect.DeepEqual(p["restart-on"], []any{"settings"}) {
|
||||
t.Errorf("restart-on = %v, want the settings file, so a changed capacity reaches the running program", p["restart-on"])
|
||||
}
|
||||
c, ok := m.Config["capacity"]
|
||||
if !ok || len(m.Config) != 1 || c.Type != "string" || c.Default != "shanks=3,ace=2,g14=0,novox=0" || c.Description == "" {
|
||||
t.Errorf("config = %+v, want one string setting, capacity, shanks=3,ace=2,g14=0,novox=0 to start", m.Config)
|
||||
}
|
||||
var settings map[string]any
|
||||
for _, r := range m.Resources {
|
||||
if r["id"] == "settings" {
|
||||
settings = r
|
||||
}
|
||||
}
|
||||
if settings == nil || settings["type"] != "file" || settings["merge"] != "json" || !reflect.DeepEqual(settings["takes-config"], []any{"capacity"}) || settings["path"] != "${dir:state}/settings.json" {
|
||||
t.Errorf("settings resource = %v", settings)
|
||||
}
|
||||
if raw, err := os.ReadFile("files/settings.json"); err != nil || !strings.Contains(string(raw), `"capacity": "shanks=3,ace=2,g14=0,novox=0"`) {
|
||||
t.Errorf("files/settings.json = %q, %v, want the setting's starting value", raw, err)
|
||||
}
|
||||
if _, bus := p["bus"]; bus {
|
||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||
}
|
||||
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "files" || m.Build.Artifacts[1].Kind != "archive" || m.Build.Artifacts[1].From != "files" {
|
||||
t.Fatalf("artifacts = %+v, want the program and the files", m.Build.Artifacts)
|
||||
}
|
||||
a := m.Build.Artifacts[0]
|
||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
|
||||
t.Errorf("artifact = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1607/AC6 (ADR 0437 §8: no timer, a wait is a context's bound)
|
||||
forbidden := []string{"Subscribe(", "QueueSubscribe", "ChanSubscribe", "time.NewTicker", "time.Tick", "time.NewTimer", "time.AfterFunc", "time.After(", "time.Sleep"}
|
||||
var files int
|
||||
for _, root := range []string{"cmd", "internal"} {
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return err
|
||||
}
|
||||
files++
|
||||
src, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
if strings.Contains(string(src), f) {
|
||||
t.Errorf("%s holds %s: the builder serves no request subject and holds no timer", path, f)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if files < 1 {
|
||||
t.Fatalf("read %d source files, want the program's", files)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
{
|
||||
"module": "mesh-delivery-builder",
|
||||
"version": "1",
|
||||
"slug": "builder",
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "builds", "history": 5 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.steps",
|
||||
"mesh-build-go.toolchains",
|
||||
"mesh-build-node.steps",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-delivery-indexer.manifests",
|
||||
"mesh-delivery-indexer.modules",
|
||||
"mesh-delivery-watchman.presence",
|
||||
"node-engine.have"
|
||||
],
|
||||
"invokes": [
|
||||
"mesh-build-go.build_go_build",
|
||||
"mesh-build-go.build_go_fetch",
|
||||
"mesh-build-node.build_node_build",
|
||||
"mesh-build-node.build_node_fetch"
|
||||
],
|
||||
"config": {
|
||||
"capacity": {
|
||||
"type": "string",
|
||||
"description": "How many build steps the builder runs at once on each node, as node=steps pairs separated by commas, such as shanks=3,ace=2 (ADR 0478 clause 3); a node set to 0 never builds, a node not named runs one at a time, and empty is one everywhere. The builder builds on shanks and ace only (the operator, 2026-10-21).",
|
||||
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
|
||||
"default": "shanks=3,ace=2,g14=0,novox=0"
|
||||
}
|
||||
},
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
},
|
||||
"secrets-owner": "mesh-delivery-builder",
|
||||
"resources": [
|
||||
{
|
||||
"id": "account",
|
||||
"type": "user",
|
||||
"name": "mesh-delivery-builder",
|
||||
"shell": "/usr/bin/nologin",
|
||||
"home": "/var/lib/mesh-delivery-builder"
|
||||
},
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"owner": "mesh-delivery-builder",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "settings",
|
||||
"trusted": false,
|
||||
"type": "file",
|
||||
"path": "${dir:state}/settings.json",
|
||||
"mode": "0600",
|
||||
"owner": "mesh-delivery-builder",
|
||||
"takes-config": ["capacity"],
|
||||
"merge": "json",
|
||||
"template": "files/settings.json"
|
||||
},
|
||||
{
|
||||
"id": "builder",
|
||||
"type": "process",
|
||||
"name": "mesh-delivery-builder",
|
||||
"artifact": "program",
|
||||
"run": [
|
||||
"./mesh-delivery-builder"
|
||||
],
|
||||
"env": {
|
||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
||||
"MESH_BUILDER_SETTINGS": "${dir:state}/settings.json"
|
||||
},
|
||||
"restart-on": [
|
||||
"settings"
|
||||
],
|
||||
"user": "mesh-delivery-builder",
|
||||
"health": {
|
||||
"kind": "unit"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "program",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/mesh-delivery-builder",
|
||||
"binary": "mesh-delivery-builder"
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -89,3 +90,34 @@ func treesVia(ctx context.Context, call caller) bridge.Trees {
|
||||
return a.Root, a.Trees, nil
|
||||
}
|
||||
}
|
||||
|
||||
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
|
||||
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
|
||||
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
|
||||
const getFileTool = "gitea.gitea_get_file"
|
||||
|
||||
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
|
||||
// recognises it as /: 404 /).
|
||||
var notFound = regexp.MustCompile(`: 404 `)
|
||||
|
||||
// filesVia asks the forge for a file through call. A file the forge says is not there is bridge.ErrNotFound.
|
||||
func filesVia(ctx context.Context, call caller) bridge.Files {
|
||||
return func(owner, repo, path, ref string) ([]byte, error) {
|
||||
raw, err := call(ctx, getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
|
||||
if err != nil {
|
||||
if notFound.MatchString(err.Error()) {
|
||||
return nil, fmt.Errorf("%w: %v", bridge.ErrNotFound, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
|
||||
}
|
||||
return []byte(*a.File.Content), nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
|
||||
// performedBucket is the module's state bucket performed, `<module>_<name>` as heads is: for each key of the gate's
|
||||
// buckets the bridge has acted on, the revision it acted at (manifest state entry performed, novox/hq ADR 0437 §8).
|
||||
// Its keys are `owed.<owed key>` and `merges.<merges key>`; its value is the revision in decimal.
|
||||
const performedBucket = delivery.ForgeBridgeModule + "_performed"
|
||||
|
||||
// kvRevisions is the bucket performed as the performer's revisions.
|
||||
type kvRevisions struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (s kvRevisions) Get(ctx context.Context, key string) (uint64, bool, error) {
|
||||
e, err := s.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
rev, err := strconv.ParseUint(string(e.Value()), 10, 64)
|
||||
if err != nil {
|
||||
return 0, false, fmt.Errorf("%s holds %q, which is no revision: %w", key, e.Value(), err)
|
||||
}
|
||||
return rev, true, nil
|
||||
}
|
||||
|
||||
func (s kvRevisions) Put(ctx context.Context, key string, rev uint64) error {
|
||||
_, err := s.kv.PutString(ctx, key, strconv.FormatUint(rev, 10))
|
||||
return err
|
||||
}
|
||||
|
||||
// revisions is what the performer keeps of the revisions it acted at.
|
||||
type revisions interface {
|
||||
Get(ctx context.Context, key string) (rev uint64, found bool, err error)
|
||||
Put(ctx context.Context, key string, rev uint64) error
|
||||
}
|
||||
|
||||
// performer does on the forge what the gate's keys ask (novox/hq ADR 0437 §8): one act for each revision of an owed
|
||||
// key, one merge for each revision of a merges key. A refusal is written on FORGE and the act is not tried again until
|
||||
// the key has a new revision; the performer holds no timer and serves no request.
|
||||
type performer struct {
|
||||
call caller
|
||||
pub publisher
|
||||
done revisions
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// seen says whether the revision of the key was acted on already. A revision that cannot be read is not seen, so the
|
||||
// act is done: a status set twice is the same status.
|
||||
func (p *performer) seen(ctx context.Context, doneKey string, rev uint64) bool {
|
||||
kept, found, err := p.done.Get(ctx, doneKey)
|
||||
if err != nil {
|
||||
p.logger.Printf("%s: the revision acted at could not be read, so the act is done again: %v", doneKey, err)
|
||||
return false
|
||||
}
|
||||
return found && kept >= rev
|
||||
}
|
||||
|
||||
func (p *performer) keep(ctx context.Context, doneKey string, rev uint64) {
|
||||
if err := p.done.Put(ctx, doneKey, rev); err != nil {
|
||||
p.logger.Printf("%s: revision %d was acted on but could not be kept, so a restart acts on it again: %v", doneKey, rev, err)
|
||||
}
|
||||
}
|
||||
|
||||
// refuse writes the refused message of an act on FORGE; it reports whether it was written. The message id makes a
|
||||
// second write of one revision's refusal the same message to the stream.
|
||||
func (p *performer) refuse(ctx context.Context, doneKey string, rev uint64, a bridge.Act, words string) bool {
|
||||
msg, err := bridge.RefusedMessage(a, words)
|
||||
if err != nil {
|
||||
p.logger.Printf("%s: the forge refused %s (%s) and the refusal cannot be said on FORGE: %v", doneKey, a.Name, words, err)
|
||||
return false
|
||||
}
|
||||
id := fmt.Sprintf("refused.%s.%d", doneKey, rev)
|
||||
if _, err := p.pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, jetstream.WithMsgID(id)); err != nil {
|
||||
p.logger.Printf("%s: the forge refused %s (%s) and FORGE could not be written: %v", doneKey, a.Name, words, err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// owed performs the act an owed key asks at revision rev, once.
|
||||
func (p *performer) owed(ctx context.Context, key string, rev uint64, value []byte) {
|
||||
doneKey := "owed." + key
|
||||
if p.seen(ctx, doneKey, rev) {
|
||||
return
|
||||
}
|
||||
a, err := bridge.OwedAct(key, value)
|
||||
if err != nil {
|
||||
p.logger.Printf("%s at revision %d is not an act the bridge can perform, and is left: %v", key, rev, err)
|
||||
p.keep(ctx, doneKey, rev)
|
||||
return
|
||||
}
|
||||
if _, err := p.call(ctx, a.Call.Tool, a.Call.Args); err != nil {
|
||||
p.logger.Printf("%s at revision %d: %s refused: %v", key, rev, a.Call.Tool, err)
|
||||
if !p.refuse(ctx, doneKey, rev, a, err.Error()) {
|
||||
return
|
||||
}
|
||||
}
|
||||
p.keep(ctx, doneKey, rev)
|
||||
}
|
||||
|
||||
// merges performs the merges a merges key hands the bridge at revision rev, once: each pull request in the state
|
||||
// merging, in the order the value names them, stopping at the first refusal (novox/hq ADR 0441 §3).
|
||||
func (p *performer) merges(ctx context.Context, key string, rev uint64, value []byte) {
|
||||
doneKey := "merges." + key
|
||||
if p.seen(ctx, doneKey, rev) {
|
||||
return
|
||||
}
|
||||
acts, err := bridge.MergeActs(value)
|
||||
if err != nil {
|
||||
p.logger.Printf("%s at revision %d cannot be merged, and is left: %v", key, rev, err)
|
||||
p.keep(ctx, doneKey, rev)
|
||||
return
|
||||
}
|
||||
for _, a := range acts {
|
||||
if _, err := p.call(ctx, a.Call.Tool, a.Call.Args); err != nil {
|
||||
p.logger.Printf("%s at revision %d: the merge of %s#%d refused: %v", key, rev, a.Repository, a.Number, err)
|
||||
if !p.refuse(ctx, doneKey, rev, a, err.Error()) {
|
||||
return
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
p.keep(ctx, doneKey, rev)
|
||||
}
|
||||
|
||||
// watch hands each key of a bucket, with the revision it has now and then each new one, to react until ctx ends or the
|
||||
// watch fails. A deleted key is not an act.
|
||||
func watch(ctx context.Context, kv jetstream.KeyValue, react func(key string, rev uint64, value []byte)) error {
|
||||
w, err := kv.Watch(ctx, ">")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return errors.New("the watch ended")
|
||||
}
|
||||
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||
continue
|
||||
}
|
||||
react(e.Key(), e.Revision(), e.Value())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
headA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
)
|
||||
|
||||
type memRevisions map[string]uint64
|
||||
|
||||
func (m memRevisions) Get(_ context.Context, key string) (uint64, bool, error) {
|
||||
r, ok := m[key]
|
||||
return r, ok, nil
|
||||
}
|
||||
func (m memRevisions) Put(_ context.Context, key string, rev uint64) error { m[key] = rev; return nil }
|
||||
|
||||
type tcall struct {
|
||||
key string
|
||||
body string
|
||||
}
|
||||
|
||||
// forgeFake records the calls and answers each from refuse, a map from a tool to its refusal.
|
||||
type forgeFake struct {
|
||||
calls []tcall
|
||||
refuse map[string]string
|
||||
}
|
||||
|
||||
func (f *forgeFake) call(_ context.Context, key string, body any) (json.RawMessage, error) {
|
||||
b, _ := json.Marshal(body)
|
||||
f.calls = append(f.calls, tcall{key, string(b)})
|
||||
if w, ok := f.refuse[key]; ok {
|
||||
return nil, errors.New(w)
|
||||
}
|
||||
return json.RawMessage(`{}`), nil
|
||||
}
|
||||
|
||||
func newPerformer(f *forgeFake) (*performer, *fakePub, memRevisions, *bytes.Buffer) {
|
||||
pub, done, logs := &fakePub{}, memRevisions{}, &bytes.Buffer{}
|
||||
return &performer{call: f.call, pub: pub, done: done, logger: log.New(logs, "", 0)}, pub, done, logs
|
||||
}
|
||||
|
||||
// The values are the SDK's own encoding (mesh-sdk go/delivery EncodeOwed, EncodeMerge), the keys its OwedKey.
|
||||
func owedKV(t *testing.T, repository string, number int, act string, o delivery.Owed) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.OwedKey(repository, number, headA, act)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := delivery.EncodeOwed(o)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, v
|
||||
}
|
||||
|
||||
func TestAnOwedStatusIsSetOnceForARevision(t *testing.T) { // 1341/AC1
|
||||
f := &forgeFake{}
|
||||
p, _, _, _ := newPerformer(f)
|
||||
key, v := owedKV(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate",
|
||||
delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "success", Text: "all green", Target: "https://forge.example/p/12"})
|
||||
p.owed(context.Background(), key, 7, v)
|
||||
if len(f.calls) != 1 || f.calls[0].key != "gitea.gitea_commit_status" ||
|
||||
f.calls[0].body != `{"context":"mesh/merge-gate","description":"all green","owner":"novox","repo":"mesh-delivery","sha":"`+headA+`","state":"success","target_url":"https://forge.example/p/12"}` {
|
||||
t.Fatalf("calls = %+v", f.calls)
|
||||
}
|
||||
p.owed(context.Background(), key, 7, v) // the same revision seen again (a restart's replay)
|
||||
if len(f.calls) != 1 {
|
||||
t.Errorf("the same revision called the tool again: %+v", f.calls)
|
||||
}
|
||||
p.owed(context.Background(), key, 8, v) // a new revision of the key is a new act
|
||||
if len(f.calls) != 2 {
|
||||
t.Errorf("a new revision was not performed: %+v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewNoteAndCommentEachCallTheirToolOnce(t *testing.T) { // 1341/AC2
|
||||
f := &forgeFake{}
|
||||
p, _, _, _ := newPerformer(f)
|
||||
for _, c := range []struct {
|
||||
act string
|
||||
o delivery.Owed
|
||||
}{
|
||||
{"view", delivery.Owed{Act: delivery.OwedView, Text: "the view"}},
|
||||
{"note", delivery.Owed{Act: delivery.OwedNote, Text: "a line", Ref: "mesh-plan"}},
|
||||
{"comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "hello"}},
|
||||
} {
|
||||
key, v := owedKV(t, "novox/mesh-delivery", 12, c.act, c.o)
|
||||
p.owed(context.Background(), key, 1, v)
|
||||
p.owed(context.Background(), key, 1, v)
|
||||
}
|
||||
var tools []string
|
||||
for _, c := range f.calls {
|
||||
tools = append(tools, c.key)
|
||||
}
|
||||
if got := strings.Join(tools, " "); got != "gitea.gitea_delivery_view gitea.gitea_note_append gitea.gitea_add_comment" {
|
||||
t.Errorf("calls = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefusedActIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1341/AC3
|
||||
f := &forgeFake{refuse: map[string]string{"gitea.gitea_add_comment": "the pull request is locked"}}
|
||||
p, pub, _, _ := newPerformer(f)
|
||||
key, v := owedKV(t, "novox/mesh-delivery", 12, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "hello"})
|
||||
p.owed(context.Background(), key, 3, v)
|
||||
p.owed(context.Background(), key, 3, v)
|
||||
if len(f.calls) != 1 {
|
||||
t.Errorf("the refused act was tried %d times at one revision", len(f.calls))
|
||||
}
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.refused" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
m, err := delivery.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != delivery.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" {
|
||||
t.Errorf("message %+v err %v", m, err)
|
||||
}
|
||||
f.refuse = nil
|
||||
p.owed(context.Background(), key, 4, v) // only a new revision tries again
|
||||
if len(f.calls) != 2 {
|
||||
t.Errorf("a new revision was not tried: %d calls", len(f.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOwedValueThatCannotBeReadIsLoggedAndNotRetried(t *testing.T) {
|
||||
f := &forgeFake{}
|
||||
p, pub, done, logs := newPerformer(f)
|
||||
p.owed(context.Background(), "not-a-key", 2, []byte(`{}`))
|
||||
if len(f.calls) != 0 || len(pub.subjects) != 0 {
|
||||
t.Errorf("calls %v published %v", f.calls, pub.subjects)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "not-a-key") {
|
||||
t.Errorf("log %q does not name the key", logs.String())
|
||||
}
|
||||
if rev, ok := done["owed.not-a-key"]; !ok || rev != 2 {
|
||||
t.Errorf("revision kept = %v %v", rev, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func mergeKV(t *testing.T, m delivery.Merge) []byte {
|
||||
t.Helper()
|
||||
v, err := delivery.EncodeMerge(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func twoMembers(a, b delivery.MergeState) delivery.Merge {
|
||||
return delivery.Merge{Group: "g", Ask: "merge.g.0123456789ab",
|
||||
Approved: map[string]string{delivery.ApprovedKey("novox/mesh-sdk", 4): headB, delivery.ApprovedKey("novox/mesh-delivery", 12): headA},
|
||||
Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, State: a, Commit: map[bool]string{true: "c1"}[a == delivery.MergeMerged]},
|
||||
{Head: delivery.Head{Repository: "novox/mesh-sdk", Number: 4, Head: headB}, State: b, Commit: map[bool]string{true: "c2"}[b == delivery.MergeMerged]},
|
||||
}}
|
||||
}
|
||||
|
||||
func TestAMergeRevisionMergesOnlyTheMemberHanded(t *testing.T) { // 1342/AC1
|
||||
f := &forgeFake{}
|
||||
p, _, _, _ := newPerformer(f)
|
||||
m := twoMembers(delivery.MergeMerging, delivery.MergeMerged)
|
||||
p.merges(context.Background(), "g", 5, mergeKV(t, m))
|
||||
want := `{"approved_heads":["novox/mesh-delivery@` + headA + `","novox/mesh-sdk@` + headB + `"],"head":"` + headA + `","number":12,"owner":"novox","repo":"mesh-delivery"}`
|
||||
if len(f.calls) != 1 || f.calls[0].key != "gitea.gitea_merge_pull_request" || f.calls[0].body != want {
|
||||
t.Fatalf("calls = %+v", f.calls)
|
||||
}
|
||||
p.merges(context.Background(), "g", 5, mergeKV(t, m))
|
||||
if len(f.calls) != 1 {
|
||||
t.Errorf("a revision merged twice: %+v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefusedMergeIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1342/AC2
|
||||
f := &forgeFake{refuse: map[string]string{"gitea.gitea_merge_pull_request": "novox/mesh-delivery#12 is not merged: its head moved"}}
|
||||
p, pub, _, _ := newPerformer(f)
|
||||
v := mergeKV(t, twoMembers(delivery.MergeMerging, delivery.MergeMerged))
|
||||
p.merges(context.Background(), "g", 5, v)
|
||||
p.merges(context.Background(), "g", 5, v)
|
||||
if len(f.calls) != 1 || len(pub.data) != 1 {
|
||||
t.Fatalf("calls %d messages %d, want one each", len(f.calls), len(pub.data))
|
||||
}
|
||||
m, err := delivery.DecodeForge(pub.data[0])
|
||||
if err != nil || m.Kind != delivery.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA ||
|
||||
m.Act != "merge" || m.Words != "novox/mesh-delivery#12 is not merged: its head moved" {
|
||||
t.Errorf("message %+v err %v", m, err)
|
||||
}
|
||||
f.refuse = nil
|
||||
p.merges(context.Background(), "g", 6, v)
|
||||
if len(f.calls) != 2 {
|
||||
t.Errorf("a new revision was not tried: %d calls", len(f.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeWithTwoHandedStopsAtTheFirstRefusal(t *testing.T) { // ADR 0441 §3
|
||||
f := &forgeFake{refuse: map[string]string{"gitea.gitea_merge_pull_request": "no"}}
|
||||
p, pub, _, _ := newPerformer(f)
|
||||
p.merges(context.Background(), "g", 1, mergeKV(t, twoMembers(delivery.MergeMerging, delivery.MergeMerging)))
|
||||
if len(f.calls) != 1 || len(pub.data) != 1 {
|
||||
t.Errorf("calls %d messages %d, want the first refusal to stop the pass", len(f.calls), len(pub.data))
|
||||
}
|
||||
}
|
||||
+238
-7
@@ -4,6 +4,8 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -25,9 +27,12 @@ import (
|
||||
// the controller's user, and the bridge's own principal <node>.<module> with the permissions the controller composes for
|
||||
// it, and no others, so a permission the bridge lacks fails a test here and not on the mesh (issue 1279).
|
||||
//
|
||||
// The gate's buckets mesh-delivery-gate_merges and mesh-delivery-gate_owed are read and watched (the manifest's reads), the
|
||||
// bucket performed is the bridge's own (its state), and the five forge tools of the gate's acts are invoked (issue 1341, 1342).
|
||||
//
|
||||
// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal
|
||||
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls and
|
||||
// gitea.gitea_commit_trees, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
|
||||
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls,
|
||||
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
|
||||
// WirePattern). The consumer's durable is <node>_<module> on EVENTS (nats.go consumerDurable). The controller's own
|
||||
// user keeps all rights, as it has.
|
||||
const busConf = `
|
||||
@@ -56,8 +61,48 @@ accounts {
|
||||
"mesh.mod.gitea.call.gitea_open_pulls.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_commit_trees"
|
||||
"mesh.mod.gitea.tool.gitea_commit_trees.*"
|
||||
"mesh.mod.gitea.call.gitea_get_file.*.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_get_file.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_get_file"
|
||||
"mesh.mod.gitea.tool.gitea_get_file.*"
|
||||
"mesh.mod.gitea.tool.gitea_open_pulls"
|
||||
"mesh.mod.gitea.tool.gitea_open_pulls.*"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_performed.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_performed.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_performed.>"
|
||||
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_performed"
|
||||
"$JS.FC.KV_mesh-delivery-forge-bridge_performed.>"
|
||||
"$KV.mesh-delivery-forge-bridge_performed.>"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.STREAM.INFO.KV_mesh-delivery-gate_merges"
|
||||
"$JS.FC.KV_mesh-delivery-gate_merges.>"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_owed.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_owed.>"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-gate_owed.>"
|
||||
"$JS.API.STREAM.INFO.KV_mesh-delivery-gate_owed"
|
||||
"$JS.FC.KV_mesh-delivery-gate_owed.>"
|
||||
"mesh.mod.gitea.call.gitea_merge_pull_request.*.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_merge_pull_request.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_merge_pull_request"
|
||||
"mesh.mod.gitea.tool.gitea_merge_pull_request.*"
|
||||
"mesh.mod.gitea.call.gitea_commit_status.*.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_commit_status.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_commit_status"
|
||||
"mesh.mod.gitea.tool.gitea_commit_status.*"
|
||||
"mesh.mod.gitea.call.gitea_delivery_view.*.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_delivery_view.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_delivery_view"
|
||||
"mesh.mod.gitea.tool.gitea_delivery_view.*"
|
||||
"mesh.mod.gitea.call.gitea_note_append.*.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_note_append.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_note_append"
|
||||
"mesh.mod.gitea.tool.gitea_note_append.*"
|
||||
"mesh.mod.gitea.call.gitea_add_comment.*.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_add_comment.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_add_comment"
|
||||
"mesh.mod.gitea.tool.gitea_add_comment.*"
|
||||
"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"
|
||||
] }
|
||||
subscribe { allow: [
|
||||
@@ -121,6 +166,21 @@ type bed struct {
|
||||
ctx context.Context
|
||||
done <-chan error
|
||||
reply func(string)
|
||||
// files sets what the stand-in forge answers to gitea_get_file from then on; it is asked with the tool's arguments.
|
||||
files func(func(owner, repo, path, ref string) (string, bool))
|
||||
}
|
||||
|
||||
// getFileReply is the reply envelope for gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile
|
||||
// answer it ({file: {path, ref, sha, size, content}}), and for a path the forge does not have the error client.ts request
|
||||
// writes: `Gitea API <path>: 404 <body>`.
|
||||
func getFileReply(path, ref, content string, found bool) []byte {
|
||||
if !found {
|
||||
b, _ := json.Marshal(map[string]any{"error": "Gitea API /repos/novox/mesh-delivery/contents/" + path + "?ref=" + ref + `: 404 {"message":"GetContentsOrList","url":"https://forge.example/api/swagger"}`})
|
||||
return b
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{"result": map[string]any{"file": map[string]any{"path": path, "ref": ref, "sha": "0123456789abcdef0123456789abcdef01234567",
|
||||
"size": len(content), "content": content}}, "node": "n0"})
|
||||
return b
|
||||
}
|
||||
|
||||
// startBed is startBridge with a step between the streams and the consumer: prepare writes what the streams hold before
|
||||
@@ -149,6 +209,13 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: headsBucket, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Its other state bucket, and the gate's buckets the bridge reads, made by the controller from the gate's manifest
|
||||
// (mesh-sdk go/delivery MergesBucket, OwedBucket with their histories).
|
||||
for name, history := range map[string]uint8{performedBucket: 1, delivery.MergesBucket: delivery.MergesHistory, delivery.OwedBucket: delivery.OwedHistory} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if prepare != nil {
|
||||
prepare(ctx, js)
|
||||
}
|
||||
@@ -176,6 +243,26 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A stand-in answers the forge's gitea_get_file: by default a module.json of {} at every path and no other file.
|
||||
var fmu sync.Mutex
|
||||
filesOf := func(_, _, path, _ string) (string, bool) { return "{}", strings.HasSuffix(path, "module.json") }
|
||||
getFile, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_get_file", func(m *nats.Msg) {
|
||||
var a struct{ Owner, Repo, Path, Ref string }
|
||||
_ = json.Unmarshal(m.Data, &a)
|
||||
fmu.Lock()
|
||||
f := filesOf
|
||||
fmu.Unlock()
|
||||
content, found := f(a.Owner, a.Repo, a.Path, a.Ref)
|
||||
_ = m.Respond(getFileReply(a.Path, a.Ref, content, found))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = getFile.Unsubscribe() })
|
||||
if err := ctl.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-forge-bridge", User: "n1.mesh-delivery-forge-bridge", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
@@ -186,7 +273,8 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(stop)
|
||||
return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() }}
|
||||
return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() },
|
||||
files: func(f func(owner, repo, path, ref string) (string, bool)) { fmu.Lock(); filesOf = f; fmu.Unlock() }}
|
||||
}
|
||||
|
||||
// TestBridgeWritesPullMessages starts the program the way the node does (its entry, run, with MESH_BROKER_FILE).
|
||||
@@ -224,7 +312,7 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true}
|
||||
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
@@ -249,8 +337,31 @@ const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
// TestBridgeWritesMergedMessage has a stand-in answer the forge's gitea_commit_trees on its tool subject
|
||||
// (mesh-control internal/broker/nats.go invokedSubjects: mesh.mod.<module>.tool.<tool>), in the reply envelope of
|
||||
// mesh-sdk go/process/call.go ({result, error, node}).
|
||||
func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4
|
||||
ctl, js, ctx, _ := startBridge(t, `{"result":{"pulls":[]}}`)
|
||||
func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
|
||||
bed := startBed(t, `{"result":{"pulls":[]}}`, nil)
|
||||
ctl, js, ctx := bed.ctl, bed.js, bed.ctx
|
||||
// The stand-in forge's files are two real files of this repository (this module's module.json and go.sum, read from the
|
||||
// working tree), in gitea_get_file's real answer shape: the root has a manifest only, modules/a a manifest and go.sum.
|
||||
manifest, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum, err := os.ReadFile("../../go.sum")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bed.files(func(_, _, path, ref string) (string, bool) {
|
||||
if ref != merge {
|
||||
t.Errorf("the forge was asked for %s at %s, want the merge commit %s", path, ref, merge)
|
||||
}
|
||||
switch path {
|
||||
case "module.json", "modules/a/module.json":
|
||||
return string(manifest), true
|
||||
case "modules/a/go.sum":
|
||||
return string(sum), true
|
||||
}
|
||||
return "", false
|
||||
})
|
||||
var asked chan []byte = make(chan []byte, 4)
|
||||
sub, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) {
|
||||
asked <- m.Data
|
||||
@@ -296,7 +407,9 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"}}
|
||||
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
|
||||
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
|
||||
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
@@ -475,3 +588,121 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBridgeMergesAGroupOneMemberAtATime: the gate (played by the test) writes merges.g with member A handed and the
|
||||
// Approve's heads of A and B; a stand-in forge merges A and announces it; the gate then hands B; both end merged on FORGE
|
||||
// (1342/AC3, ADR 0437 §8's check).
|
||||
func TestBridgeMergesAGroupOneMemberAtATime(t *testing.T) { // 1342/AC3
|
||||
b := startBed(t, `{"result":{"pulls":[]}}`, nil)
|
||||
const headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
const mergeA, mergeB = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222"
|
||||
merges := make(chan string, 4)
|
||||
// The stand-in for gitea_merge_pull_request answers in the reply envelope of mesh-sdk go/process/call.go and publishes
|
||||
// gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 does, with the merge commit of the member merged.
|
||||
sub, err := b.ctl.Subscribe("mesh.mod.gitea.tool.gitea_merge_pull_request", func(m *nats.Msg) {
|
||||
var a struct {
|
||||
Repo string `json:"repo"`
|
||||
Head string `json:"head"`
|
||||
}
|
||||
_ = json.Unmarshal(m.Data, &a)
|
||||
merges <- string(m.Data)
|
||||
commit, number := mergeA, 12
|
||||
if a.Repo == "mesh-sdk" {
|
||||
commit, number = mergeB, 4
|
||||
}
|
||||
ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": a.Repo, "number": number, "head_sha": a.Head, "head": "feat/x",
|
||||
"base": "main", "merge_commit_sha": commit, "module_dirs": []string{"modules/a"}, "module_dirs_said": true})
|
||||
_, _ = b.js.Publish(b.ctx, "mesh.mod.gitea.event.pull.merged", ev)
|
||||
_ = m.Respond([]byte(`{"result":{"merged":true},"node":"n0"}`))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
trees, err := b.ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) {
|
||||
_ = m.Respond([]byte(`{"result":{"root":"r0","trees":{"":"t0","modules/a":"t1"}},"node":"n0"}`))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = trees.Unsubscribe() })
|
||||
if err := b.ctl.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
kv, err := b.js.KeyValue(b.ctx, delivery.MergesBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
approved := map[string]string{delivery.ApprovedKey("novox/mesh-delivery", 12): head, delivery.ApprovedKey("novox/mesh-sdk", 4): headB}
|
||||
pullA := delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: head}, State: delivery.MergeMerging}
|
||||
pullB := delivery.MergePull{Head: delivery.Head{Repository: "novox/mesh-sdk", Number: 4, Head: headB}, State: delivery.MergeMerging}
|
||||
put := func(pulls ...delivery.MergePull) {
|
||||
v, err := delivery.EncodeMerge(delivery.Merge{Group: "g", Ask: "merge.g.0123456789ab", Approved: approved, Pulls: pulls})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(b.ctx, "g", v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
next := func(repo string) {
|
||||
select {
|
||||
case got := <-merges:
|
||||
if !strings.Contains(got, `"repo":"`+repo+`"`) {
|
||||
t.Errorf("the forge was asked %s, want %s", got, repo)
|
||||
}
|
||||
for _, want := range []string{"novox/mesh-delivery@" + head, "novox/mesh-sdk@" + headB} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("the merge %s does not carry the approved head %s", got, want)
|
||||
}
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("%s was not merged within 10 s", repo)
|
||||
}
|
||||
}
|
||||
put(pullA)
|
||||
next("mesh-delivery")
|
||||
// The gate hands the next member only after the bridge reported the one before merged (ADR 0441 §5).
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && len(forgeMerged(t, b)) < 1; time.Sleep(50 * time.Millisecond) {
|
||||
}
|
||||
pullA.State, pullA.Commit = delivery.MergeMerged, mergeA
|
||||
put(pullA, pullB)
|
||||
next("mesh-sdk")
|
||||
var got []string
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if got = forgeMerged(t, b); len(got) == 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("FORGE holds merged messages for %v, want both members", got)
|
||||
}
|
||||
select {
|
||||
case extra := <-merges:
|
||||
t.Errorf("a third merge was asked: %s", extra)
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// forgeMerged lists the repositories FORGE holds a merged message for, sorted.
|
||||
func forgeMerged(t *testing.T, b *bed) []string {
|
||||
t.Helper()
|
||||
forge, err := b.js.Stream(b.ctx, "FORGE")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out []string
|
||||
for _, repo := range []string{"mesh-delivery", "mesh-sdk"} {
|
||||
raw, err := forge.GetLastMsgForSubject(b.ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox."+repo+".merged")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if f, err := delivery.DecodeForge(raw.Data); err == nil && f.Kind == delivery.ForgeMerged {
|
||||
out = append(out, f.Repository)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sha256Sum(b []byte) []byte { s := sha256.Sum256(b); return s[:] }
|
||||
|
||||
@@ -115,8 +115,8 @@ type publisher interface {
|
||||
// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never
|
||||
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, logger *log.Logger) {
|
||||
msg, err := bridge.Convert(m.Subject(), m.Data(), trees)
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) {
|
||||
msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files)
|
||||
switch {
|
||||
case errors.Is(err, bridge.ErrRetry):
|
||||
logger.Printf("%v; heard again in %s", err, retryDelay)
|
||||
@@ -196,6 +196,35 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Everything written on FORGE, an event's and a sync's, keeps the head in the bucket.
|
||||
rec := &recorder{pub: js, heads: heads, logger: logger}
|
||||
said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) }
|
||||
// What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision.
|
||||
performedKV, err := js.KeyValue(ctx, performedBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", performedBucket, err)
|
||||
}
|
||||
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
|
||||
watching := make(chan error, 2)
|
||||
for _, w := range []struct {
|
||||
bucket string
|
||||
react func(ctx context.Context, key string, rev uint64, value []byte)
|
||||
}{{delivery.OwedBucket, perf.owed}, {delivery.MergesBucket, perf.merges}} {
|
||||
kv, err := js.KeyValue(ctx, w.bucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
// The gate's buckets come with the gate; the bridge does not wait for them (no timer), and is started again
|
||||
// by its delivery when the gate arrives.
|
||||
logger.Printf("the bucket %s does not exist yet: nothing of it is performed until the bridge starts again", w.bucket)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", w.bucket, err)
|
||||
}
|
||||
go func() {
|
||||
watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) {
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
w.react(hctx, key, rev, value)
|
||||
})
|
||||
}()
|
||||
}
|
||||
if err := said(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -209,7 +238,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Bounded: the turn is held, and no sync or other event goes on without it.
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
handle(hctx, m, rec, treesVia(hctx, call), logger)
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger)
|
||||
})
|
||||
},
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
@@ -217,6 +246,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
}
|
||||
defer cc.Stop()
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case err := <-watching:
|
||||
return fmt.Errorf("watching the gate's keys: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +16,8 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
|
||||
const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
@@ -64,7 +68,7 @@ func body(m map[string]any) []byte {
|
||||
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
@@ -80,7 +84,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
|
||||
func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
@@ -95,7 +99,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
|
||||
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
if m.acked != 0 || m.termd != 0 {
|
||||
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
|
||||
}
|
||||
@@ -117,7 +121,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("the tree listing of novox/mesh-delivery was cut")
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
@@ -136,13 +140,19 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{}
|
||||
var keys []string
|
||||
var args []any
|
||||
var mu sync.Mutex // the files are asked for concurrently (issue 1458)
|
||||
call := func(_ context.Context, key string, body any) (json.RawMessage, error) {
|
||||
mu.Lock()
|
||||
keys, args = append(keys, key), append(args, body)
|
||||
mu.Unlock()
|
||||
if key == "gitea.gitea_get_file" { // the manifests, 1448; the trees are what this test is about
|
||||
return json.RawMessage(`{"file":{"path":"p","sha":"0123","size":2,"content":"{}"}}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
if len(keys) != 1 || keys[0] != "gitea.gitea_commit_trees" {
|
||||
t.Fatalf("calls %v", keys)
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 {
|
||||
t.Fatalf("calls %v, want the trees once and first", keys)
|
||||
}
|
||||
if got, _ := json.Marshal(args[0]); string(got) != `{"owner":"novox","repo":"mesh-delivery","sha":"89abcdef0123456789abcdef0123456789abcdef"}` {
|
||||
t.Errorf("arguments %s", got)
|
||||
@@ -171,3 +181,52 @@ func TestAnAnswerWithoutARootIsNotUsed(t *testing.T) { // 1236 (a shape the tool
|
||||
t.Error("an answer that is not an object was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// The answer of gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile write it, and the
|
||||
// error of a path the forge does not have as client.ts request writes it (`Gitea API ${path}: ${status} ${text}`).
|
||||
const getFileAnswer = `{"file":{"path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","sha":"0123","size":4,"content":"x y\n"}}`
|
||||
const getFile404 = "Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=89abcdef: 404 {\"message\":\"GetContentsOrList\",\"url\":\"https://forge.example/api/swagger\"}"
|
||||
|
||||
func TestFilesViaAsksByOwnerRepoPathRefAndTakesTheContent(t *testing.T) { // 1448/AC4
|
||||
var keys []string
|
||||
var args []any
|
||||
call := func(_ context.Context, key string, body any) (json.RawMessage, error) {
|
||||
keys, args = append(keys, key), append(args, body)
|
||||
return json.RawMessage(getFileAnswer), nil
|
||||
}
|
||||
got, err := filesVia(context.Background(), call)("novox", "mesh-delivery", "modules/a/go.sum", "89abcdef0123456789abcdef0123456789abcdef")
|
||||
if err != nil || string(got) != "x y\n" {
|
||||
t.Fatalf("got %q err %v", got, err)
|
||||
}
|
||||
if len(keys) != 1 || keys[0] != "gitea.gitea_get_file" {
|
||||
t.Fatalf("calls %v", keys)
|
||||
}
|
||||
if b, _ := json.Marshal(args[0]); string(b) != `{"owner":"novox","path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","repo":"mesh-delivery"}` {
|
||||
t.Errorf("arguments %s", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesViaTellsNotFoundFromOtherErrorsAndOtherShapes(t *testing.T) { // 1448/AC3
|
||||
call := func(err error, raw string) caller {
|
||||
return func(context.Context, string, any) (json.RawMessage, error) { return json.RawMessage(raw), err }
|
||||
}
|
||||
if _, err := filesVia(context.Background(), call(errors.New(getFile404), ""))("o", "r", "p", "s"); !errors.Is(err, bridge.ErrNotFound) {
|
||||
t.Errorf("a 404 is %v, want ErrNotFound", err)
|
||||
}
|
||||
if _, err := filesVia(context.Background(), call(errors.New("Gitea API /repos/o/r/contents/p: 502 bad gateway"), ""))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) {
|
||||
t.Errorf("a 502 is %v, want an error that is not ErrNotFound", err)
|
||||
}
|
||||
for _, raw := range []string{`{"file":null}`, `{}`, `[1]`, `{"file":{"path":"p"}}`} {
|
||||
if _, err := filesVia(context.Background(), call(nil, raw))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) {
|
||||
t.Errorf("the answer %s gave %v, want an error", raw, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// someFiles is a forge that has a module.json at every path and no other file, for the tests that are not about the files.
|
||||
func someFiles(_, _, path, _ string) ([]byte, error) {
|
||||
if strings.HasSuffix(path, "module.json") {
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
return nil, bridge.ErrNotFound
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9 h1:XL1dbtSleGAixg6FX4gEnSG/AJxRG/tWw6/7U1RuzHc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Call is a call of a forge tool: the tool by its manifest key and its arguments.
|
||||
type Call struct {
|
||||
Tool string
|
||||
Args map[string]any
|
||||
}
|
||||
|
||||
// Act is one thing the bridge does on the forge for a key of the gate: the act's name (an owed key's act, `status=<context>`,
|
||||
// `view`, `note`, `comment.<seq>`, or `merge`), the pull request and head it is on (Number 0 for a commit no pull
|
||||
// request names) and the forge call that performs it.
|
||||
type Act struct {
|
||||
Name string
|
||||
Repository string
|
||||
Number int
|
||||
Head string
|
||||
Call Call
|
||||
}
|
||||
|
||||
// The forge's tools, by manifest key; their arguments are the ones of mesh-catalog modules/gitea: delivery.ts
|
||||
// commitStatusTool (owner, repo, sha, context, state, description, target_url), tools/index.ts gitea_delivery_view
|
||||
// (owner, repo, number, body), gitea_note_append (owner, repo, sha, ref, line), gitea_add_comment (owner, repo,
|
||||
// number, body), merge.ts gitea_merge_pull_request (owner, repo, number, head, approved_heads).
|
||||
const (
|
||||
commitStatusTool = "gitea.gitea_commit_status"
|
||||
deliveryViewTool = "gitea.gitea_delivery_view"
|
||||
noteAppendTool = "gitea.gitea_note_append"
|
||||
addCommentTool = "gitea.gitea_add_comment"
|
||||
mergePullTool = "gitea.gitea_merge_pull_request"
|
||||
)
|
||||
|
||||
// MergeAct is the name of the act of merging a pull request.
|
||||
const MergeAct = "merge"
|
||||
|
||||
func ownerRepo(repository string) (owner, repo string) {
|
||||
owner, repo, _ = strings.Cut(repository, "/")
|
||||
return owner, repo
|
||||
}
|
||||
|
||||
// OwedAct turns an owed key and its value (mesh-sdk go/delivery OwedKey, EncodeOwed) into the forge call that performs
|
||||
// it. A key that is no owed key, a value that is no owed act, a value whose act is not the key's, and a view or a comment
|
||||
// on no pull request are errors, each saying why.
|
||||
func OwedAct(key string, value []byte) (Act, error) {
|
||||
repository, number, head, name, err := delivery.ParseOwedKey(key)
|
||||
if err != nil {
|
||||
return Act{}, err
|
||||
}
|
||||
o, err := delivery.DecodeOwed(value)
|
||||
if err != nil {
|
||||
return Act{}, fmt.Errorf("the owed act %s: %w", key, err)
|
||||
}
|
||||
kind, context, err := delivery.ParseOwedAct(name)
|
||||
if err != nil {
|
||||
return Act{}, err
|
||||
}
|
||||
if o.Act != kind {
|
||||
return Act{}, fmt.Errorf("the owed act %s says %q, not the %q its key names", key, o.Act, kind)
|
||||
}
|
||||
owner, repo := ownerRepo(repository)
|
||||
a := Act{Name: name, Repository: repository, Number: number, Head: head}
|
||||
needsPull := func() error {
|
||||
if number <= 0 {
|
||||
return fmt.Errorf("the owed %s %s is on a commit no pull request names, and it needs the pull request", kind, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
switch kind {
|
||||
case delivery.OwedStatus:
|
||||
if o.Context != context {
|
||||
return Act{}, fmt.Errorf("the owed status %s says the context %q, not the %q its key names", key, o.Context, context)
|
||||
}
|
||||
a.Call = Call{Tool: commitStatusTool, Args: map[string]any{"owner": owner, "repo": repo, "sha": head, "context": o.Context,
|
||||
"state": o.State, "description": o.Text, "target_url": o.Target}}
|
||||
case delivery.OwedView:
|
||||
if err := needsPull(); err != nil {
|
||||
return Act{}, err
|
||||
}
|
||||
a.Call = Call{Tool: deliveryViewTool, Args: map[string]any{"owner": owner, "repo": repo, "number": number, "body": o.Text}}
|
||||
case delivery.OwedNote:
|
||||
a.Call = Call{Tool: noteAppendTool, Args: map[string]any{"owner": owner, "repo": repo, "sha": head, "ref": o.Ref, "line": o.Text}}
|
||||
case delivery.OwedComment:
|
||||
if err := needsPull(); err != nil {
|
||||
return Act{}, err
|
||||
}
|
||||
a.Call = Call{Tool: addCommentTool, Args: map[string]any{"owner": owner, "repo": repo, "number": number, "body": o.Text}}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// MergeActs turns the value of a merges key (mesh-sdk go/delivery EncodeMerge) into the merges to perform: one for each
|
||||
// pull request in the state merging, in the order the value names them, at the head it expects and with the approved
|
||||
// heads of every member of the group, `<owner>/<repo>@<full sha>` sorted (the merge tool's approved_heads). A pull
|
||||
// request merged or refused already gets none. A value with no approved heads is an error: the forge's merge tool
|
||||
// needs them for a group.
|
||||
func MergeActs(value []byte) ([]Act, error) {
|
||||
m, err := delivery.DecodeMerge(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var handed []delivery.MergePull
|
||||
for _, p := range m.Pulls {
|
||||
if p.State == delivery.MergeMerging {
|
||||
handed = append(handed, p)
|
||||
}
|
||||
}
|
||||
if len(handed) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if len(m.Approved) == 0 {
|
||||
return nil, fmt.Errorf("the merge of the group %s names no approved heads, so the forge's merge tool has nothing to merge at", m.Group)
|
||||
}
|
||||
approved := make([]string, 0, len(m.Approved))
|
||||
for key, sha := range m.Approved {
|
||||
repository, _, ok := strings.Cut(key, "@")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the approved head %q of the group %s is not <repository>@<number>", key, m.Group)
|
||||
}
|
||||
approved = append(approved, repository+"@"+sha)
|
||||
}
|
||||
slices.Sort(approved)
|
||||
var acts []Act
|
||||
for _, p := range handed {
|
||||
owner, repo := ownerRepo(p.Repository)
|
||||
acts = append(acts, Act{Name: MergeAct, Repository: p.Repository, Number: p.Number, Head: p.Head.Head,
|
||||
Call: Call{Tool: mergePullTool, Args: map[string]any{"owner": owner, "repo": repo, "number": p.Number, "head": p.Head.Head,
|
||||
"approved_heads": approved}}})
|
||||
}
|
||||
return acts, nil
|
||||
}
|
||||
|
||||
// RefusedMessage is the refused message the bridge writes on FORGE when the forge refuses an act: the act, the pull
|
||||
// request and head it was on, and the forge's own words.
|
||||
func RefusedMessage(a Act, words string) (Message, error) {
|
||||
event, err := delivery.ForgeEvent(a.Repository, delivery.ForgeRefused)
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeRefused, Repository: a.Repository, Number: a.Number,
|
||||
Head: a.Head, Act: a.Name, Words: words})
|
||||
if err != nil {
|
||||
return Message{}, err
|
||||
}
|
||||
return Message{Event: event, Data: body}, nil
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
headA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
)
|
||||
|
||||
// owedValue is an owed value as the SDK writes it (mesh-sdk go/delivery EncodeOwed), and its key as OwedKey spells it.
|
||||
func owedValue(t *testing.T, repository string, number int, act string, o delivery.Owed) (string, []byte) {
|
||||
t.Helper()
|
||||
key, err := delivery.OwedKey(repository, number, headA, act)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := delivery.EncodeOwed(o)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, b
|
||||
}
|
||||
|
||||
func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1
|
||||
key, v := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate",
|
||||
delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "pending", Text: "waiting", Target: "https://forge.example/novox/mesh-delivery/pulls/12"})
|
||||
a, err := OwedAct(key, v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := Call{Tool: "gitea.gitea_commit_status", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "sha": headA,
|
||||
"context": "mesh/merge-gate", "state": "pending", "description": "waiting", "target_url": "https://forge.example/novox/mesh-delivery/pulls/12"}}
|
||||
if !reflect.DeepEqual(a.Call, want) {
|
||||
t.Errorf("call = %+v, want %+v", a.Call, want)
|
||||
}
|
||||
if a.Repository != "novox/mesh-delivery" || a.Number != 12 || a.Head != headA || a.Name != "status=mesh/merge-gate" {
|
||||
t.Errorf("act = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedViewNoteAndCommentCalls(t *testing.T) { // 1341/AC2
|
||||
cases := []struct {
|
||||
act string
|
||||
o delivery.Owed
|
||||
want Call
|
||||
}{
|
||||
{"view", delivery.Owed{Act: delivery.OwedView, Text: "the view"},
|
||||
Call{Tool: "gitea.gitea_delivery_view", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "body": "the view"}}},
|
||||
{"note", delivery.Owed{Act: delivery.OwedNote, Text: "a line", Ref: "mesh-plan"},
|
||||
Call{Tool: "gitea.gitea_note_append", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "sha": headA, "ref": "mesh-plan", "line": "a line"}}},
|
||||
{"comment.2", delivery.Owed{Act: delivery.OwedComment, Text: "hello"},
|
||||
Call{Tool: "gitea.gitea_add_comment", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "body": "hello"}}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
key, v := owedValue(t, "novox/mesh-delivery", 12, c.act, c.o)
|
||||
a, err := OwedAct(key, v)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", c.act, err)
|
||||
}
|
||||
if !reflect.DeepEqual(a.Call, c.want) {
|
||||
t.Errorf("%s: call = %+v, want %+v", c.act, a.Call, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedActWithoutAPullRequestNeedsOneOnlyWhereTheToolDoes(t *testing.T) {
|
||||
key, v := owedValue(t, "novox/mesh-delivery", 0, "note", delivery.Owed{Act: delivery.OwedNote, Text: "l", Ref: "mesh-plan"})
|
||||
if _, err := OwedAct(key, v); err != nil {
|
||||
t.Errorf("a note on a commit no pull request names: %v", err)
|
||||
}
|
||||
key, v = owedValue(t, "novox/mesh-delivery", 0, "view", delivery.Owed{Act: delivery.OwedView, Text: "v"})
|
||||
if _, err := OwedAct(key, v); err == nil {
|
||||
t.Error("a view of no pull request was taken")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedActOfADottedRepositoryIsUnescaped(t *testing.T) {
|
||||
key, v := owedValue(t, "novox/my.repo", 3, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "x"})
|
||||
a, err := OwedAct(key, v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" {
|
||||
t.Errorf("act = %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOwedActThatDisagreesWithItsKeyIsRefused(t *testing.T) {
|
||||
key, _ := owedValue(t, "novox/mesh-delivery", 12, "view", delivery.Owed{Act: delivery.OwedView, Text: "v"})
|
||||
v, _ := delivery.EncodeOwed(delivery.Owed{Act: delivery.OwedComment, Text: "v"})
|
||||
if _, err := OwedAct(key, v); err == nil {
|
||||
t.Error("a comment under a view's key was taken")
|
||||
}
|
||||
if _, err := OwedAct("nonsense", v); err == nil {
|
||||
t.Error("a key that is no owed key was taken")
|
||||
}
|
||||
if _, err := OwedAct(key, []byte(`{"v":1`)); err == nil {
|
||||
t.Error("a value that is not json was taken")
|
||||
}
|
||||
}
|
||||
|
||||
func mergeValue(t *testing.T, m delivery.Merge) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeMerge(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestMergeActsHandOnlyTheMemberHandedWithEveryApprovedHead(t *testing.T) { // 1342/AC1
|
||||
approved := map[string]string{delivery.ApprovedKey("novox/mesh-sdk", 4): headB, delivery.ApprovedKey("novox/mesh-delivery", 12): headA}
|
||||
v := mergeValue(t, delivery.Merge{Group: "g", Ask: "merge.g.0123456789ab", Approved: approved, Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/mesh-sdk", Number: 4, Head: headB}, State: delivery.MergeMerged, Commit: "c0"},
|
||||
{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, State: delivery.MergeMerging},
|
||||
}})
|
||||
acts, err := MergeActs(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(acts) != 1 {
|
||||
t.Fatalf("%d acts, want 1 (the merged member is not merged again)", len(acts))
|
||||
}
|
||||
want := Call{Tool: "gitea.gitea_merge_pull_request", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "head": headA,
|
||||
"approved_heads": []string{"novox/mesh-delivery@" + headA, "novox/mesh-sdk@" + headB}}}
|
||||
if !reflect.DeepEqual(acts[0].Call, want) {
|
||||
t.Errorf("call = %+v, want %+v", acts[0].Call, want)
|
||||
}
|
||||
if acts[0].Name != "merge" || acts[0].Number != 12 || acts[0].Head != headA {
|
||||
t.Errorf("act = %+v", acts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeActsOfAMergeWithoutApprovedHeadsAreRefused(t *testing.T) {
|
||||
v := mergeValue(t, delivery.Merge{Group: "g", Ask: "a", Pulls: []delivery.MergePull{
|
||||
{Head: delivery.Head{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, State: delivery.MergeMerging}}})
|
||||
if _, err := MergeActs(v); err == nil {
|
||||
t.Error("a merge with no approved heads was taken: the forge's tool would refuse it with nothing to say which")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefusedMessageNamesTheActAndTheForgesWords(t *testing.T) { // 1341/AC3, 1342/AC2
|
||||
a := Act{Name: "merge", Repository: "novox/mesh-delivery", Number: 12, Head: headA}
|
||||
m, err := RefusedMessage(a, "novox/mesh-delivery#12 is not merged: its head moved")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Event != "forge.novox.mesh-delivery.refused" {
|
||||
t.Errorf("event = %q", m.Event)
|
||||
}
|
||||
f, err := delivery.DecodeForge(m.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA,
|
||||
Act: "merge", Words: "novox/mesh-delivery#12 is not merged: its head moved"}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v", f)
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,17 @@
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -34,6 +38,17 @@ const Trunk = "main"
|
||||
// (the root directory spelled ""), as the forge's gitea_commit_trees answers (issue 1234/AC4).
|
||||
type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error)
|
||||
|
||||
// ErrNotFound says the forge has no file at that path and ref (the forge's "404", issue 1448).
|
||||
var ErrNotFound = errors.New("the forge has no such file")
|
||||
|
||||
// Files asks the forge for one file's bytes at a ref (a commit), as the forge's gitea_get_file answers; a file the forge
|
||||
// says is not there is ErrNotFound, any other failure any other error (issue 1448).
|
||||
type Files func(owner, repo, path, ref string) ([]byte, error)
|
||||
|
||||
// LockFiles are the lock files read beside a module's manifest, by name relative to its directory (novox/hq ADR 0465
|
||||
// clause 1: go.sum for a Go module, package-lock.json for a Node one).
|
||||
var LockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// Message is a message for FORGE: the event name (forge.<owner>.<name>.<kind>) and its JSON body.
|
||||
type Message struct {
|
||||
Event string
|
||||
@@ -72,7 +87,7 @@ func unreadable(subject, why string, args ...any) error {
|
||||
// is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into
|
||||
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
|
||||
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
|
||||
func Convert(subject string, data []byte, trees Trees) (Message, error) {
|
||||
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
|
||||
var action string
|
||||
switch subject {
|
||||
case UpdatedSubject:
|
||||
@@ -99,13 +114,17 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) {
|
||||
return Message{}, unreadable(subject, "no branch")
|
||||
}
|
||||
if subject == MergedSubject && p.Base == Trunk {
|
||||
return merged(subject, p, trees)
|
||||
return merged(subject, p, trees, files)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: action}
|
||||
// The directories are the forge's word only when it said it could read them (module_dirs_said).
|
||||
if action == delivery.PullUpdated && p.ModuleDirsSaid {
|
||||
f.Dirs, f.DirsKnown = p.ModuleDirs, true
|
||||
var err error
|
||||
if f.Manifests, f.Locks, err = inputsOf(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
|
||||
}
|
||||
}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
if err != nil {
|
||||
@@ -118,8 +137,97 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) {
|
||||
return Message{Event: event, Data: body}, nil
|
||||
}
|
||||
|
||||
// MaxReadsInFlight is how many files of the forge inputsOf asks for at once. A merge into a repository of many
|
||||
// modules (131 in the catalogue) reads three files for each, one after another inside the bound of a handled event
|
||||
// (30 s) that leaves about 76 ms a call; eight at once keeps the forge's load small and the read well inside the bound
|
||||
// (issue 1458).
|
||||
const MaxReadsInFlight = 8
|
||||
|
||||
// read is one file asked for: the path in the directory dir, and whether it is the module's manifest.
|
||||
type read struct {
|
||||
dir, path string
|
||||
manifest bool
|
||||
raw []byte
|
||||
err error
|
||||
}
|
||||
|
||||
// inputsOf reads, at ref, the module.json of each directory (the root spelled "") and the lock files beside it
|
||||
// (LockFiles): the manifests as the forge answered them and the lock files' hex sha256 by name. A manifest the forge
|
||||
// does not have is an error, since the directory was said to hold one; a lock file it does not have is left out; any other
|
||||
// failure is an error (issue 1448).
|
||||
//
|
||||
// The files are asked for with at most MaxReadsInFlight calls at once (issue 1458). The result does not depend on the
|
||||
// order the calls finish in, and the error is that of the first failing file in the order of dirs and LockFiles. After
|
||||
// a failure no further file is asked for, and inputsOf returns only when every call it started has returned.
|
||||
func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) {
|
||||
at := func(dir, name string) string {
|
||||
if dir == "" {
|
||||
return name
|
||||
}
|
||||
return dir + "/" + name
|
||||
}
|
||||
var reads []*read
|
||||
for _, dir := range dirs {
|
||||
reads = append(reads, &read{dir: dir, path: at(dir, "module.json"), manifest: true})
|
||||
for _, name := range LockFiles {
|
||||
reads = append(reads, &read{dir: dir, path: at(dir, name)})
|
||||
}
|
||||
}
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
failed bool
|
||||
slots = make(chan struct{}, MaxReadsInFlight)
|
||||
)
|
||||
for _, r := range reads {
|
||||
slots <- struct{}{}
|
||||
mu.Lock()
|
||||
stop := failed
|
||||
mu.Unlock()
|
||||
if stop {
|
||||
<-slots
|
||||
break
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(r *read) {
|
||||
defer func() { <-slots; wg.Done() }()
|
||||
r.raw, r.err = files(owner, repo, r.path, ref)
|
||||
if r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)) {
|
||||
mu.Lock()
|
||||
failed = true
|
||||
mu.Unlock()
|
||||
}
|
||||
}(r)
|
||||
}
|
||||
wg.Wait()
|
||||
manifests = map[string]string{}
|
||||
locks = map[string]map[string]string{}
|
||||
for _, r := range reads {
|
||||
switch {
|
||||
case r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)):
|
||||
return nil, nil, fmt.Errorf("%s: %w", r.path, r.err)
|
||||
case r.err != nil: // a lock file the forge does not have
|
||||
case r.manifest:
|
||||
manifests[r.dir] = string(r.raw)
|
||||
default:
|
||||
sum := sha256.Sum256(r.raw)
|
||||
if locks[r.dir] == nil {
|
||||
locks[r.dir] = map[string]string{}
|
||||
}
|
||||
locks[r.dir][path.Base(r.path)] = hex.EncodeToString(sum[:])
|
||||
}
|
||||
}
|
||||
if len(manifests) == 0 {
|
||||
manifests = nil
|
||||
}
|
||||
if len(locks) == 0 {
|
||||
locks = nil
|
||||
}
|
||||
return manifests, locks, nil
|
||||
}
|
||||
|
||||
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
|
||||
func merged(subject string, p pull, trees Trees) (Message, error) {
|
||||
func merged(subject string, p pull, trees Trees, files Files) (Message, error) {
|
||||
if !sha40.MatchString(p.MergeSHA) {
|
||||
return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
|
||||
}
|
||||
@@ -131,12 +239,21 @@ func merged(subject string, p pull, trees Trees) (Message, error) {
|
||||
if len(dirs) == 0 {
|
||||
return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s", ErrNoModules, subject, repository, p.MergeSHA)
|
||||
}
|
||||
names := make([]string, 0, len(dirs))
|
||||
for dir := range dirs {
|
||||
names = append(names, dir)
|
||||
}
|
||||
slices.Sort(names)
|
||||
manifests, locks, err := inputsOf(p.Owner, p.Repo, p.MergeSHA, names, files)
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
|
||||
}
|
||||
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs})
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks})
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ const updatedSubject = "mesh.mod.gitea.event.pull.updated"
|
||||
const closedSubject = "mesh.mod.gitea.event.pull.closed"
|
||||
|
||||
func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; the bus is the integration test)
|
||||
msg, err := Convert(updatedSubject, updated(nil), nil)
|
||||
msg, err := Convert(updatedSubject, updated(nil), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -59,14 +59,14 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion;
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true}
|
||||
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
msg, err := Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil)
|
||||
msg, err := Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -75,7 +75,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
// Said and empty is known and empty, not unknown.
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil)
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -84,7 +84,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
// Dirs listed without the forge saying it could read them are not trusted.
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil)
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -94,7 +94,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
}
|
||||
|
||||
func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
|
||||
msg, err := Convert(closedSubject, closed(), nil)
|
||||
msg, err := Convert(closedSubject, closed(), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -130,7 +130,7 @@ func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5
|
||||
"closed no head": {closedSubject, mustDrop(closed(), "head_sha"), "head"},
|
||||
"closed no repo": {closedSubject, mustDrop(closed(), "repo"), "repo"},
|
||||
} {
|
||||
msg, err := Convert(c.subject, c.data, nil)
|
||||
msg, err := Convert(c.subject, c.data, nil, anyFiles)
|
||||
if err == nil {
|
||||
t.Errorf("%s: a message %q was made", name, msg.Event)
|
||||
continue
|
||||
@@ -157,7 +157,7 @@ func mustDrop(body []byte, key string) []byte {
|
||||
|
||||
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235
|
||||
for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} {
|
||||
if _, err := Convert(s, updated(nil), nil); !errors.Is(err, ErrNotHandled) {
|
||||
if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) {
|
||||
t.Errorf("%s: err = %v, want ErrNotHandled", s, err)
|
||||
}
|
||||
}
|
||||
@@ -200,7 +200,7 @@ func treesOf(calls *[]call, root string, trees map[string]string, err error) Tre
|
||||
func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
|
||||
var calls []call
|
||||
answer := map[string]string{"modules/a": "t1", "modules/b": "t2"}
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil))
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil), anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -212,7 +212,7 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Merge: merge, Root: "r0", Trees: answer}
|
||||
Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
@@ -223,7 +223,7 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
|
||||
|
||||
func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil))
|
||||
msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -243,7 +243,7 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
|
||||
|
||||
func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the conversion; the delay is in main_test)
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")))
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")), anyFiles)
|
||||
if !errors.Is(err, ErrRetry) || msg.Data != nil {
|
||||
t.Fatalf("err = %v data = %s, want ErrRetry and no message", err, msg.Data)
|
||||
}
|
||||
@@ -256,7 +256,7 @@ func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the co
|
||||
|
||||
func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (notes: a repository with no module.json)
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil))
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil), anyFiles)
|
||||
if !errors.Is(err, ErrNoModules) || msg.Data != nil {
|
||||
t.Errorf("err = %v data = %s", err, msg.Data)
|
||||
}
|
||||
@@ -264,7 +264,7 @@ func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (note
|
||||
|
||||
func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236
|
||||
var calls []call
|
||||
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil))
|
||||
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
|
||||
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
|
||||
t.Errorf("err = %v calls = %v", err, calls)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the
|
||||
// module.json and the go.sum of modules/mesh-delivery-forge-bridge (two directories up). The forge's own answer shape
|
||||
// is mesh-catalog modules/gitea tools/index.ts gitea_get_file; the bridge sees only the bytes (issue 1448).
|
||||
func real(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile("../../" + name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func hexSum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
|
||||
|
||||
type fileCall struct{ owner, repo, path, ref string }
|
||||
|
||||
// forgeFiles answers as the forge does for a path: its bytes, ErrNotFound for a path in none, the error in errs.
|
||||
type forgeFiles struct {
|
||||
have map[string][]byte
|
||||
errs map[string]error
|
||||
mu sync.Mutex
|
||||
calls []fileCall
|
||||
}
|
||||
|
||||
func (f *forgeFiles) get(owner, repo, path, ref string) ([]byte, error) {
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, fileCall{owner, repo, path, ref})
|
||||
f.mu.Unlock()
|
||||
if err, ok := f.errs[path]; ok {
|
||||
return nil, err
|
||||
}
|
||||
if b, ok := f.have[path]; ok {
|
||||
return b, nil
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
// anyFiles is a forge that has a module.json at every path and no lock file, for tests that are not about the files.
|
||||
func anyFiles(owner, repo, path, ref string) ([]byte, error) {
|
||||
if strings.HasSuffix(path, "module.json") {
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
manifest, sum := real(t, "module.json"), real(t, "go.sum")
|
||||
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}}
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := map[string]string{"modules/a": string(manifest), "modules/b": `{"module":"b"}`}; !reflect.DeepEqual(f.Manifests, want) {
|
||||
t.Errorf("manifests = %v", f.Manifests)
|
||||
}
|
||||
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
|
||||
t.Errorf("locks = %v, want %v", f.Locks, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
manifest := real(t, "module.json")
|
||||
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest}}
|
||||
msg, err := Convert(updatedSubject, updated(nil), nil, forge.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ := delivery.DecodeForge(msg.Data)
|
||||
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 0 {
|
||||
t.Errorf("manifests %v locks %v", f.Manifests, f.Locks)
|
||||
}
|
||||
if len(forge.calls) == 0 || forge.calls[0].ref != head {
|
||||
t.Errorf("calls %v, want the head %s", forge.calls, head)
|
||||
}
|
||||
// Dirs not known: neither field, and the forge is not asked.
|
||||
forge = &forgeFiles{}
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, forge.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ = delivery.DecodeForge(msg.Data)
|
||||
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
|
||||
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3
|
||||
boom := errors.New("Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=x: 502 bad gateway")
|
||||
trees := func() Trees { var c []call; return treesOf(&c, "r0", map[string]string{"modules/a": "t1"}, nil) }
|
||||
for name, forge := range map[string]*forgeFiles{
|
||||
"a manifest error": {errs: map[string]error{"modules/a/module.json": boom}},
|
||||
"a manifest not found": {},
|
||||
"a lock file error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/go.sum": boom}},
|
||||
"a second lock error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/package-lock.json": boom}},
|
||||
} {
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
|
||||
if !errors.Is(err, ErrRetry) || msg.Data != nil {
|
||||
t.Errorf("%s: err = %v data = %s, want ErrRetry and no message", name, err, msg.Data)
|
||||
}
|
||||
_, err = Convert(updatedSubject, updated(nil), nil, forge.get)
|
||||
if !errors.Is(err, ErrRetry) {
|
||||
t.Errorf("%s (pull): err = %v, want ErrRetry", name, err)
|
||||
}
|
||||
}
|
||||
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil {
|
||||
t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFilesAreAskedAtTheMergeCommitByOwnerRepoPathAndRef(t *testing.T) { // 1448/AC4 (the call's arguments; the manifest is in manifest_test)
|
||||
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
|
||||
var calls []call
|
||||
if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []fileCall{{"novox", "mesh-delivery", "modules/a/module.json", merge}, {"novox", "mesh-delivery", "modules/a/go.sum", merge},
|
||||
{"novox", "mesh-delivery", "modules/a/package-lock.json", merge}}
|
||||
sort.Slice(forge.calls, func(i, j int) bool { return forge.calls[i].path < forge.calls[j].path }) // they are asked for concurrently
|
||||
sort.Slice(want, func(i, j int) bool { return want[i].path < want[j].path })
|
||||
if !reflect.DeepEqual(forge.calls, want) {
|
||||
t.Errorf("calls = %v, want %v", forge.calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
// slowForge answers like a forge that takes d a call: a module.json of the directory's name everywhere, a go.sum in
|
||||
// the directories whose number is even and in the root, and counts the calls in flight.
|
||||
type slowForge struct {
|
||||
d time.Duration
|
||||
mu sync.Mutex
|
||||
inFlight, most, started, n int
|
||||
failOn string
|
||||
finished int
|
||||
}
|
||||
|
||||
func (f *slowForge) get(_, _, path, _ string) ([]byte, error) {
|
||||
f.mu.Lock()
|
||||
f.inFlight++
|
||||
f.started++
|
||||
if f.inFlight > f.most {
|
||||
f.most = f.inFlight
|
||||
}
|
||||
f.mu.Unlock()
|
||||
time.Sleep(f.d)
|
||||
defer func() { f.mu.Lock(); f.inFlight--; f.finished++; f.mu.Unlock() }()
|
||||
switch {
|
||||
case path == f.failOn:
|
||||
return nil, errors.New("Gitea API " + path + ": 502 bad gateway")
|
||||
case strings.HasSuffix(path, "module.json"):
|
||||
return []byte(`{"module":"` + path + `"}`), nil
|
||||
case strings.HasSuffix(path, "go.sum") && (path == "go.sum" || path[len(path)-8]%2 == 0):
|
||||
return []byte("sum of " + path), nil
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
func manyDirs(n int) []string {
|
||||
dirs := []string{""}
|
||||
for i := 1; i < n; i++ {
|
||||
dirs = append(dirs, fmt.Sprintf("modules/m%02d", i))
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
func TestFortyDirectoriesAreReadEightAtATimeWithTheResultOfOneAtATime(t *testing.T) { // 1458/AC1
|
||||
dirs := manyDirs(40)
|
||||
slow := &slowForge{d: 10 * time.Millisecond}
|
||||
manifests, locks, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if slow.most > MaxReadsInFlight || slow.most < 2 {
|
||||
t.Errorf("at most %d calls were in flight, want more than one and at most %d", slow.most, MaxReadsInFlight)
|
||||
}
|
||||
if slow.started != 40*3 {
|
||||
t.Errorf("%d calls, want 120", slow.started)
|
||||
}
|
||||
// The same read one file at a time.
|
||||
wantM, wantL := map[string]string{}, map[string]map[string]string{}
|
||||
one := &slowForge{}
|
||||
for _, dir := range dirs {
|
||||
m, l, err := inputsOfOne(dir, one.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantM[dir] = m
|
||||
if l != nil {
|
||||
wantL[dir] = l
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(manifests, wantM) || !reflect.DeepEqual(locks, wantL) {
|
||||
t.Errorf("the concurrent read differs from a one-at-a-time read: %d manifests %d locks, want %d and %d", len(manifests), len(locks), len(wantM), len(wantL))
|
||||
}
|
||||
if len(locks) == 0 || len(locks) == 40 {
|
||||
t.Errorf("%d directories with locks, the fixture should have some and not all", len(locks))
|
||||
}
|
||||
}
|
||||
|
||||
// inputsOfOne reads one directory with a single call at a time, as the bridge did before issue 1458.
|
||||
func inputsOfOne(dir string, files Files) (string, map[string]string, error) {
|
||||
at := func(name string) string {
|
||||
if dir == "" {
|
||||
return name
|
||||
}
|
||||
return dir + "/" + name
|
||||
}
|
||||
raw, err := files("o", "r", at("module.json"), "x")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var locks map[string]string
|
||||
for _, name := range LockFiles {
|
||||
b, err := files("o", "r", at(name), "x")
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if locks == nil {
|
||||
locks = map[string]string{}
|
||||
}
|
||||
s := sha256.Sum256(b)
|
||||
locks[name] = hex.EncodeToString(s[:])
|
||||
}
|
||||
return string(raw), locks, nil
|
||||
}
|
||||
|
||||
func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testing.T) { // 1458/AC2
|
||||
dirs := manyDirs(40)
|
||||
slow := &slowForge{d: 10 * time.Millisecond, failOn: "modules/m05/go.sum"}
|
||||
_, _, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
|
||||
if err == nil || !strings.Contains(err.Error(), "modules/m05/go.sum") || !strings.Contains(err.Error(), "502") {
|
||||
t.Fatalf("err = %v, want the failing file named", err)
|
||||
}
|
||||
slow.mu.Lock()
|
||||
defer slow.mu.Unlock()
|
||||
if slow.inFlight != 0 || slow.finished != slow.started {
|
||||
t.Errorf("%d calls in flight, %d of %d finished, when the read returned", slow.inFlight, slow.finished, slow.started)
|
||||
}
|
||||
if slow.started >= 40*3 {
|
||||
t.Errorf("%d calls were made, want the read to stop asking after the failure", slow.started)
|
||||
}
|
||||
// Through Convert it is ErrRetry.
|
||||
var calls []call
|
||||
dirsMap := map[string]string{}
|
||||
for _, d := range dirs[1:] {
|
||||
dirsMap[d] = "t"
|
||||
}
|
||||
slow2 := &slowForge{failOn: "modules/m05/module.json"}
|
||||
if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", dirsMap, nil), slow2.get); !errors.Is(err, ErrRetry) {
|
||||
t.Errorf("err = %v, want ErrRetry", err)
|
||||
}
|
||||
}
|
||||
@@ -93,7 +93,7 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view",
|
||||
"gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
|
||||
"gitea.gitea_get_file", "gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
@@ -102,9 +102,11 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
|
||||
if m.SecretsOwner != "mesh-delivery-forge-bridge" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// One bucket: the head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443 §5).
|
||||
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}}) { // 1279/AC6
|
||||
t.Errorf("state = %+v, want the one bucket heads", m.State)
|
||||
// Two buckets: the head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443
|
||||
// §5), and the revision of each key of the gate's buckets the bridge performed, so a key is performed once per
|
||||
// revision (ADR 0437 §8; issue 1341).
|
||||
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}}) { // 1279/AC6, 1341
|
||||
t.Errorf("state = %+v, want the buckets heads and performed", m.State)
|
||||
}
|
||||
// No other state, no bus, no seat, no tool (ADR 0437 §4).
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
|
||||
|
||||
@@ -11,7 +11,8 @@
|
||||
{ "name": "forge", "events": ["forge.**"] }
|
||||
],
|
||||
"state": [
|
||||
{ "name": "heads", "history": 1 }
|
||||
{ "name": "heads", "history": 1 },
|
||||
{ "name": "performed", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.merges",
|
||||
@@ -24,7 +25,8 @@
|
||||
"gitea.gitea_note_append",
|
||||
"gitea.gitea_add_comment",
|
||||
"gitea.gitea_open_pulls",
|
||||
"gitea.gitea_commit_trees"
|
||||
"gitea.gitea_commit_trees",
|
||||
"gitea.gitea_get_file"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
|
||||
@@ -46,7 +46,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
|
||||
for _, b := range []struct{ repo, module, path, id string }{
|
||||
{"novox/mesh-delivery", "a", "modules/a", "build-1"}, {"novox/mesh-catalog", "c", "modules/c", "build-2"},
|
||||
} {
|
||||
rec, _ := json.Marshal(map[string]any{"id": b.id, "state": "built", "repository": b.repo, "ref": "main", "module": b.module,
|
||||
rec, _ := json.Marshal(map[string]any{"id": b.id, "repository": b.repo, "ref": "main", "module": b.module,
|
||||
"commit": "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", "path": b.path, "at": "2026-10-19T09:38:00Z"})
|
||||
if _, err := builds.Put(ctx, delivery.BuildKey(b.module, b.id), rec); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -95,7 +95,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10
|
||||
|
||||
// startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill
|
||||
// mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created.
|
||||
func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
|
||||
func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) {
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -137,6 +137,15 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) (
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, "FORGE", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The build modules' streams for toolchain.reported (declared by them, heard by the indexer) and its durable consumers.
|
||||
for module, stream := range map[string]string{"mesh-build-go": "TOOLCHAINS_GO", "mesh-build-node": "TOOLCHAINS_NODE"} {
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.stream." + module + ".event.toolchain.reported"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
candidates, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.CandidatesBucket, History: delivery.CandidatesHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -145,11 +154,18 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) (
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449).
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket, History: delivery.ManifestsHistory}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuildsBucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put(ctx, builds)
|
||||
for _, b := range before {
|
||||
b(ctx, js)
|
||||
}
|
||||
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-indexer", User: "n1.mesh-delivery-indexer", Password: "pw"})
|
||||
@@ -172,7 +188,7 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9
|
||||
// go/delivery/testdata/build-record-build-agent.json, captured from the controller's encoder).
|
||||
const inputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
|
||||
rec := `{"id":"build-1","state":"built","repository":"novox/mesh-delivery","ref":"main","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/a","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
|
||||
rec := `{"id":"build-1","repository":"novox/mesh-delivery","ref":"main","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/a","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
|
||||
if _, err := builds.Put(ctx, delivery.BuildKey("a", "build-1"), []byte(rec)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -199,7 +215,139 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9
|
||||
}
|
||||
}
|
||||
}
|
||||
if m.FP != want || m.Repository != "novox/mesh-delivery" || m.Commit != merge {
|
||||
t.Errorf("module = %+v within 5 s, want fp %s", m, want)
|
||||
if m.FP != want || m.Repository != "novox/mesh-delivery" || m.Commit != merge || m.Path != "modules/a" { // 1606/AC2
|
||||
t.Errorf("module = %+v within 5 s, want fp %s and path modules/a", m, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the
|
||||
// hash of its go.sum (the bridge's Manifests and Locks, issue 1448) makes, within 5 s, modules.avahi with its manifest digest,
|
||||
// its inputs and the reason it has no fingerprint of the new form, and manifests.<digest> with the manifest byte for byte.
|
||||
func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4
|
||||
// The manifest is avahi's module.json from mesh-catalog at a67f01963f0d9ef0913a63720b89e573ca7a16ef (copied into the
|
||||
// index package's testdata); the lock hash is the sha256 of modules/avahi/go.sum there.
|
||||
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||
const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
|
||||
rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
|
||||
if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
Locks: map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m delivery.Module
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := modules.Get(ctx, "avahi"); err == nil {
|
||||
if m, err = delivery.DecodeModule(e.Value()); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
digest := delivery.ManifestDigest(manifest)
|
||||
if m.Manifest != digest || m.None2 != "toolchain not yet reported" || m.FP2 != "" || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" ||
|
||||
m.Inputs2.Locks["tools"] != goSum || m.Path != "modules/avahi" { // 1606/AC2
|
||||
t.Errorf("module = %+v within 5 s, want its path modules/avahi, its manifest digest %s, its inputs and none2 'toolchain not yet reported'", m, digest)
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e, err := kv.Get(ctx, digest)
|
||||
if err != nil {
|
||||
t.Fatalf("manifests.%s: %v", digest, err)
|
||||
}
|
||||
if got, err := delivery.DecodeManifest(e.Value()); err != nil || got.Module != "avahi" || got.JSON != string(manifest) {
|
||||
t.Errorf("manifest = %+v err %v, want avahi and the file's bytes", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain: the bucket mesh-build-go_toolchains exists (the state
|
||||
// toolchains of the build module, delivery.ToolchainsBucket) and holds no key; a merged message of mesh-catalog carrying
|
||||
// avahi's real manifest makes modules.avahi with none2 'toolchain not yet reported'; the bucket made after the indexer started, a put of the key n1.go and the build module's event toolchain.reported (the build
|
||||
// module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 5 s and with no merge, none2 "builder's
|
||||
// fingerprint not yet known".
|
||||
func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *testing.T) { // 1444/AC10
|
||||
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||
const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
|
||||
rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
|
||||
if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
Locks: map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
none2 := func(want string) string {
|
||||
var got string
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := modules.Get(ctx, "avahi"); err == nil {
|
||||
if m, err := delivery.DecodeModule(e.Value()); err == nil {
|
||||
if got = m.None2; got == want {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return got
|
||||
}
|
||||
if got := none2(delivery.ToolchainNoReport); got != delivery.ToolchainNoReport {
|
||||
t.Fatalf("none2 = %q within 5 s, want %q", got, delivery.ToolchainNoReport)
|
||||
}
|
||||
// The bucket is made only now, as when the build module is assigned after the indexer started (review finding 9): the
|
||||
// indexer finds it on its own.
|
||||
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := delivery.ToolchainKey("n1", "go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The build module says it wrote the key (its event on its stream): the indexer then watches the bucket it did not know.
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-build-go.event.toolchain.reported", []byte(`{"node":"n1","language":"go"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := none2("builder's fingerprint not yet known"); got != "builder's fingerprint not yet known" {
|
||||
t.Errorf("none2 = %q within 5 s of the toolchain key, want the builder's fingerprint not yet known", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,13 +251,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := bucket(delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
builds, err := bucket(delivery.BuildsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The gate's source bucket (mesh-delivery-gate_source) is read from release 3d on; it does not exist before, and the
|
||||
// indexer does not wait for it.
|
||||
|
||||
known := index.NewBuilds()
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
@@ -273,7 +274,49 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
errs := make(chan error, 1)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Builds: known, Logf: logger.Printf}
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf}
|
||||
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||
sourceErrs := make(chan error, 1)
|
||||
switch source, err := js.KeyValue(ctx, delivery.SourceBucket); {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
logger.Printf("the bucket %s does not exist yet: no source revision is read until the indexer starts again", delivery.SourceBucket)
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
|
||||
default:
|
||||
each := func(e jetstream.KeyValueEntry) {
|
||||
if opOf(e) {
|
||||
applySource(ctx, ix, e.Key(), e.Revision(), e.Value(), logger)
|
||||
}
|
||||
}
|
||||
sw, err := settle(ctx, source, each)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
||||
}
|
||||
tools := newToolchainWatch(ctx, js, toolchainSink{put: ix.PutToolchain, apply: ix.ApplyToolchain, recompute: ix.Recompute}, logger)
|
||||
if err := tools.start(); err != nil {
|
||||
return err
|
||||
}
|
||||
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
|
||||
// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
|
||||
// with the build module, as with the gate's source.
|
||||
for _, module := range sortedModules() {
|
||||
stream := index.ToolchainStreams[module]
|
||||
tcons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the indexer starts again: %v", cred.Node, cred.Module, stream, module, err)
|
||||
continue
|
||||
}
|
||||
tc, err := tcons.Consume(func(m jetstream.Msg) { tools.Reported(m) },
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tc.Stop()
|
||||
}
|
||||
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err)
|
||||
@@ -289,5 +332,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return nil
|
||||
case err := <-errs:
|
||||
return err
|
||||
case err := <-sourceErrs:
|
||||
return err
|
||||
case err := <-tools.errs:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,3 +67,24 @@ func TestAMessageThatIsNoForgeMessageIsTerminatedAndLogged(t *testing.T) { // 12
|
||||
t.Errorf("terminated %d acked %d keys %d log %q", m.termd, m.acked, len(st.vals), logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSourceMainKeyIsTakenAtItsRevision(t *testing.T) { // 1344/AC1 (the watch's side: key, revision, value)
|
||||
var logs bytes.Buffer
|
||||
logger := log.New(&logs, "", 0)
|
||||
mods := &memStore{vals: map[string][]byte{}}
|
||||
ix := &index.Indexer{Store: &memStore{vals: map[string][]byte{}}, Modules: mods, Builds: index.NewBuilds()}
|
||||
value, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{"novox/mesh-delivery": head}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applySource(context.Background(), ix, "other", 3, value, logger) // not the key of the mesh's source
|
||||
applySource(context.Background(), ix, delivery.SourceMainKey, 4, []byte(`{"v":1`), logger)
|
||||
if !strings.Contains(logs.String(), "source") {
|
||||
t.Errorf("a source that cannot be read was not said: %q", logs.String())
|
||||
}
|
||||
logs.Reset()
|
||||
applySource(context.Background(), ix, delivery.SourceMainKey, 5, value, logger)
|
||||
if logs.Len() != 0 {
|
||||
t.Errorf("a good source said %q", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// applySource takes one put of the gate's bucket source into the indexer: the key main, at its revision, which is the mesh's
|
||||
// source revision (mesh-sdk go/delivery SourceMainKey, Source). Any other key is not the mesh's source. A value that is no
|
||||
// source is said; so is a write that failed, and it is tried again only at the next revision (novox/hq ADR 0437 §8).
|
||||
func applySource(ctx context.Context, ix *index.Indexer, key string, rev uint64, value []byte, logger *log.Logger) {
|
||||
if key != delivery.SourceMainKey {
|
||||
return
|
||||
}
|
||||
s, err := delivery.DecodeSource(value)
|
||||
if err != nil {
|
||||
logger.Printf("source.%s at revision %d is not a source, and is left: %v", key, rev, err)
|
||||
return
|
||||
}
|
||||
if err := ix.ApplySource(ctx, rev, s); err != nil {
|
||||
logger.Printf("source.%s at revision %d could not be taken into the modules: %v", key, rev, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-indexer/internal/index"
|
||||
)
|
||||
|
||||
// toolchainSink is where the toolchains the build modules report go: the indexer's own methods (index.Indexer.PutToolchain,
|
||||
// ApplyToolchain, Recompute).
|
||||
type toolchainSink struct {
|
||||
put func(module, key string, value []byte, deleted bool) bool
|
||||
apply func(ctx context.Context, module, key string, value []byte, deleted bool) error
|
||||
recompute func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// bucketOpener is the part of the bus the toolchain watch uses.
|
||||
type bucketOpener interface {
|
||||
KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error)
|
||||
}
|
||||
|
||||
// toolchainWatch reads the toolchains each build module reports (mesh-build-go_toolchains, mesh-build-node_toolchains: the
|
||||
// module's state toolchains, delivery.ToolchainsBucket; novox/hq ADR 0465 clause 2, issue 1444) and follows them: every
|
||||
// change of a language's agreed toolchain writes the modules of that language again (index.Indexer.Recompute). A bucket
|
||||
// is made when its build module is assigned, after the indexer started, and the indexer holds no timer (ADR 0437 §8):
|
||||
// it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
|
||||
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched, settled as the others.
|
||||
type toolchainWatch struct {
|
||||
ctx context.Context
|
||||
js bucketOpener
|
||||
sink toolchainSink
|
||||
logger *log.Logger
|
||||
errs chan error // the error of a watch that ended
|
||||
mu sync.Mutex
|
||||
watched map[string]bool
|
||||
}
|
||||
|
||||
func newToolchainWatch(ctx context.Context, js bucketOpener, sink toolchainSink, logger *log.Logger) *toolchainWatch {
|
||||
return &toolchainWatch{ctx: ctx, js: js, sink: sink, logger: logger, errs: make(chan error, len(index.ToolchainModules)), watched: map[string]bool{}}
|
||||
}
|
||||
|
||||
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
|
||||
func (w *toolchainWatch) start() error {
|
||||
for _, module := range sortedModules() {
|
||||
if err := w.ensure(module); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensure watches the module's toolchains bucket if it is not watched and exists (settling what it holds), then writes the
|
||||
// modules again from what the watches hold.
|
||||
func (w *toolchainWatch) ensure(module string) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
name := delivery.ToolchainsBucket(module)
|
||||
if !w.watched[module] {
|
||||
kv, err := w.js.KeyValue(w.ctx, name)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
|
||||
return nil
|
||||
case err != nil:
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.sink.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.watched[module] = true
|
||||
go func() {
|
||||
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
|
||||
if err := w.sink.apply(w.ctx, module, e.Key(), e.Value(), !opOf(e)); err != nil {
|
||||
w.logger.Printf("%s.%s could not be taken into the modules: %v", name, e.Key(), err)
|
||||
}
|
||||
})
|
||||
}()
|
||||
}
|
||||
return w.sink.recompute(w.ctx)
|
||||
}
|
||||
|
||||
// Reported takes one toolchain.reported event of a build module (the message on its stream) in: the module is read off the
|
||||
// subject, mesh.stream.<module>.event.toolchain.reported, its bucket is watched if it is not, and the modules are written
|
||||
// again. A subject of no build module is terminated; a failure stays unacknowledged, and the server delivers it again.
|
||||
func (w *toolchainWatch) Reported(m delivered) {
|
||||
module, _, _ := strings.Cut(strings.TrimPrefix(m.Subject(), "mesh.stream."), ".event.")
|
||||
known := false
|
||||
for _, k := range index.ToolchainModules {
|
||||
known = known || k == module
|
||||
}
|
||||
if !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") {
|
||||
w.logger.Printf("%s: not a build module's event, dropped", m.Subject())
|
||||
_ = m.Term()
|
||||
return
|
||||
}
|
||||
if err := w.ensure(module); err != nil {
|
||||
w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err)
|
||||
return
|
||||
}
|
||||
_ = m.Ack()
|
||||
}
|
||||
|
||||
func sortedModules() []string { return slices.Sorted(maps.Values(index.ToolchainModules)) }
|
||||
@@ -0,0 +1,141 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// A build module's bucket is made when the module is assigned, after the indexer started (issue 1444, review finding 9).
|
||||
// The fakes implement only what the watch calls: the bus's KeyValue by bucket name, and a bucket's WatchAll.
|
||||
|
||||
type laterBuckets struct {
|
||||
mu sync.Mutex
|
||||
appears map[string]jetstream.KeyValue // buckets that exist now
|
||||
asked map[string]int
|
||||
}
|
||||
|
||||
func (l *laterBuckets) KeyValue(_ context.Context, name string) (jetstream.KeyValue, error) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.asked[name]++
|
||||
if kv, ok := l.appears[name]; ok {
|
||||
return kv, nil
|
||||
}
|
||||
return nil, jetstream.ErrBucketNotFound
|
||||
}
|
||||
|
||||
func (l *laterBuckets) make(name string, kv jetstream.KeyValue) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.appears[name] = kv
|
||||
}
|
||||
|
||||
type fakeBucket struct {
|
||||
jetstream.KeyValue
|
||||
held []jetstream.KeyValueEntry
|
||||
w *fakeWatcher
|
||||
}
|
||||
|
||||
func (b *fakeBucket) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
||||
ch := make(chan jetstream.KeyValueEntry, 8)
|
||||
for _, e := range b.held {
|
||||
ch <- e
|
||||
}
|
||||
ch <- nil // the mark of the last value held
|
||||
b.w = &fakeWatcher{ch: ch}
|
||||
return b.w, nil
|
||||
}
|
||||
|
||||
type fakeWatcher struct {
|
||||
jetstream.KeyWatcher
|
||||
ch chan jetstream.KeyValueEntry
|
||||
}
|
||||
|
||||
func (w *fakeWatcher) Updates() <-chan jetstream.KeyValueEntry { return w.ch }
|
||||
func (w *fakeWatcher) Stop() error { return nil }
|
||||
|
||||
type fakeEntry struct {
|
||||
jetstream.KeyValueEntry
|
||||
key string
|
||||
value []byte
|
||||
}
|
||||
|
||||
func (e fakeEntry) Key() string { return e.key }
|
||||
func (e fakeEntry) Value() []byte { return e.value }
|
||||
func (e fakeEntry) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePut }
|
||||
|
||||
func TestABuildModulesBucketThatAppearsLaterIsWatchedWhenItsEventArrives(t *testing.T) { // 1444 (review finding 9)
|
||||
ctx, stop := context.WithCancel(context.Background())
|
||||
defer stop()
|
||||
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buckets := &laterBuckets{appears: map[string]jetstream.KeyValue{}, asked: map[string]int{}}
|
||||
var mu sync.Mutex
|
||||
var puts, applies []string
|
||||
recomputes := 0
|
||||
sink := toolchainSink{
|
||||
put: func(module, key string, _ []byte, _ bool) bool {
|
||||
mu.Lock()
|
||||
puts = append(puts, module+" "+key)
|
||||
mu.Unlock()
|
||||
return true
|
||||
},
|
||||
apply: func(_ context.Context, module, key string, _ []byte, _ bool) error {
|
||||
mu.Lock()
|
||||
applies = append(applies, module+" "+key)
|
||||
mu.Unlock()
|
||||
return nil
|
||||
},
|
||||
recompute: func(context.Context) error { mu.Lock(); recomputes++; mu.Unlock(); return nil },
|
||||
}
|
||||
var logs bytes.Buffer
|
||||
w := newToolchainWatch(ctx, buckets, sink, log.New(&logs, "", 0))
|
||||
if err := w.start(); err != nil { // neither bucket exists: nothing is watched, nothing is written, nobody waits
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(puts) != 0 || recomputes != 0 || !strings.Contains(logs.String(), "does not exist yet") {
|
||||
t.Fatalf("puts %v recomputes %d log %q after a start with no bucket", puts, recomputes, logs.String())
|
||||
}
|
||||
// The bucket of mesh-build-go is made and already holds a key, and the module says it reported.
|
||||
bucket := &fakeBucket{held: []jetstream.KeyValueEntry{fakeEntry{key: "novox.go", value: value}}}
|
||||
buckets.make(delivery.ToolchainsBucket("mesh-build-go"), bucket)
|
||||
msg := &fakeMsg{subject: "mesh.stream.mesh-build-go.event.toolchain.reported"}
|
||||
w.Reported(msg)
|
||||
if msg.acked != 1 || len(puts) != 1 || puts[0] != "mesh-build-go novox.go" || recomputes != 1 {
|
||||
t.Fatalf("acked %d puts %v recomputes %d, want the settled key, one recompute and an ack", msg.acked, puts, recomputes)
|
||||
}
|
||||
// It is followed from now on: a key put later is applied, and a second report only recomputes.
|
||||
bucket.w.ch <- fakeEntry{key: "ace.go", value: value}
|
||||
for deadline := time.Now().Add(3 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) {
|
||||
mu.Lock()
|
||||
n := len(applies)
|
||||
mu.Unlock()
|
||||
if n == 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(applies) != 1 || applies[0] != "mesh-build-go ace.go" {
|
||||
t.Errorf("applies = %v, want the key put later", applies)
|
||||
}
|
||||
w.Reported(&fakeMsg{subject: "mesh.stream.mesh-build-go.event.toolchain.reported"})
|
||||
if len(puts) != 1 || recomputes != 2 || buckets.asked[delivery.ToolchainsBucket("mesh-build-go")] != 2 {
|
||||
t.Errorf("puts %v recomputes %d asked %v, want the bucket opened again only at the first report", puts, recomputes, buckets.asked)
|
||||
}
|
||||
// An event of no build module is terminated.
|
||||
other := &fakeMsg{subject: "mesh.stream.avahi.event.toolchain.reported"}
|
||||
w.Reported(other)
|
||||
if other.termd != 1 || other.acked != 0 {
|
||||
t.Errorf("terminated %d acked %d for a module that is no build module", other.termd, other.acked)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9 h1:XL1dbtSleGAixg6FX4gEnSG/AJxRG/tWw6/7U1RuzHc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -57,7 +57,11 @@ func (b *Builds) Delete(key string) {
|
||||
}
|
||||
}
|
||||
|
||||
func dirKey(repository, dir string) string { return repository + "\x00" + strings.Trim(dir, "/") }
|
||||
// dirKey names a directory of a repository whichever way the repository is spelled: a build record names it by its clone
|
||||
// URL, a forge message by "owner/name" (issue 1281).
|
||||
func dirKey(repository, dir string) string {
|
||||
return strings.ToLower(delivery.RepositoryName(repository)) + "\x00" + strings.Trim(dir, "/")
|
||||
}
|
||||
|
||||
// Module is the module a build record of the repository names the directory as its path, if one does.
|
||||
func (b *Builds) Module(repository, dir string) (string, bool) {
|
||||
@@ -85,8 +89,8 @@ func (b *Builds) ModuleOf(repository, dir string) string {
|
||||
return path.Base(strings.Trim(dir, "/"))
|
||||
}
|
||||
|
||||
// Known is a module the builds know of in a repository: its directory there and its newest successful trunk build (ref
|
||||
// main, state built, with source inputs) if any. The newest key of a module is any state and ref, so it is not what
|
||||
// Known is a module the builds know of in a repository: its directory there and its newest successful trunk build
|
||||
// (delivery.BuildRecord.IsTrunkBuild) if any. The newest key of a module is any state and ref, so it is not what
|
||||
// finds this (mesh-sdk go/delivery/buildrecord.go NewestKey); the records are.
|
||||
type Known struct {
|
||||
Module string
|
||||
@@ -115,7 +119,7 @@ func (b *Builds) Modules(repository string) []Known {
|
||||
} else if newer(r, m.any) {
|
||||
m.any = r
|
||||
}
|
||||
if r.Ref == "main" && r.State == "built" && r.SourceInputs != nil && (m.trunk == nil || newer(r, *m.trunk)) {
|
||||
if r.IsTrunkBuild() && (m.trunk == nil || newer(r, *m.trunk)) {
|
||||
c := r
|
||||
m.trunk = &c
|
||||
}
|
||||
@@ -127,7 +131,7 @@ func (b *Builds) Modules(repository string) []Known {
|
||||
if m.trunk != nil {
|
||||
k.Repository, k.Path = m.trunk.Repository, strings.Trim(m.trunk.Path, "/")
|
||||
}
|
||||
if k.Repository == repository {
|
||||
if delivery.SameRepository(k.Repository, repository) {
|
||||
out = append(out, k)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func manifestPull(t *testing.T, number int, head, manifest string) delivery.Forge {
|
||||
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
|
||||
if manifest != "" {
|
||||
f.Manifests = map[string]string{"modules/avahi": manifest}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func candidateOf(t *testing.T, st *memStore) delivery.Candidate {
|
||||
t.Helper()
|
||||
key, err := delivery.CandidatesKey("feat/x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, ok := st.vals[key]
|
||||
if !ok {
|
||||
t.Fatalf("candidates holds %d keys, none %s", len(st.vals), key)
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t *testing.T) { // 1451/AC1
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check
|
||||
want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"}
|
||||
if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || c.Pulls[0].Fps["avahi"] != want {
|
||||
t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want)
|
||||
}
|
||||
if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) {
|
||||
t.Errorf("fpset = %q, want %q", c.FPSet, delivery.FingerprintSet(c.Pulls))
|
||||
}
|
||||
// The same pull request heard again without its directories (the head unchanged) keeps what it had.
|
||||
again := manifestPull(t, 301, h1, "")
|
||||
again.Dirs, again.DirsKnown = nil, false
|
||||
if err := ix.Apply(context.Background(), again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c2 := candidateOf(t, ix.Store.(*memStore)); c2.Pulls[0].Fps["avahi"] != want || c2.FPSet != c.FPSet {
|
||||
t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/AC2
|
||||
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := candidateOf(t, ix.Store.(*memStore))
|
||||
if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.FPSet != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" {
|
||||
t.Errorf("candidate = %+v, want the module avahi and no Fps and no fpset", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
// The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory.
|
||||
other := catalogueManifest(t, "ca-trust")
|
||||
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := candidateOf(t, ix.Store.(*memStore))
|
||||
writes := ix.Store.(*memStore).writes
|
||||
err := ix.Apply(context.Background(), manifestPull(t, 302, h2, other))
|
||||
if !errors.Is(err, ErrUnreadable) {
|
||||
t.Fatalf("err = %v, want a refusal", err)
|
||||
}
|
||||
for _, w := range []string{catalogueRepo + "#301", catalogueRepo + "#302", `"avahi"`} {
|
||||
if !strings.Contains(err.Error(), w) {
|
||||
t.Errorf("the refusal %q does not name %s", err, w)
|
||||
}
|
||||
}
|
||||
// Candidate.check refuses the group too, which is what the refusal stands for.
|
||||
if _, encErr := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat.x", HeadsHash: before.HeadsHash, Pulls: []delivery.CandidatePull{before.Pulls[0], {
|
||||
Head: delivery.Head{Repository: catalogueRepo, Number: 302, Head: h2}, Modules: []string{"avahi"},
|
||||
Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(other)), None2: NoneNoPreview}}}}}); encErr == nil {
|
||||
t.Error("the candidate check accepts two manifests for one module")
|
||||
}
|
||||
if ix.Store.(*memStore).writes != writes {
|
||||
t.Error("a partial write was made")
|
||||
}
|
||||
if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.FPSet != before.FPSet {
|
||||
t.Errorf("the candidate changed: %+v", after)
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -42,8 +43,15 @@ type Indexer struct {
|
||||
Store Store // candidates
|
||||
// Modules is the modules bucket; nil writes no fingerprints.
|
||||
Modules Store
|
||||
Builds *Builds
|
||||
Logf func(format string, args ...any)
|
||||
// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
|
||||
Manifests Store
|
||||
Builds *Builds
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex // guards source and toolchains, which the watches and the forge messages all reach
|
||||
source sourceNow
|
||||
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
|
||||
recompute sync.Mutex // one Recompute at a time
|
||||
}
|
||||
|
||||
func (ix *Indexer) logf(format string, args ...any) {
|
||||
@@ -68,6 +76,13 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var digests map[string]string
|
||||
if f.Kind == delivery.ForgePull {
|
||||
var err error
|
||||
if digests, err = ix.keepManifests(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if trunkLike[f.Branch] {
|
||||
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
|
||||
return nil
|
||||
@@ -85,7 +100,12 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
}
|
||||
cur = &c
|
||||
}
|
||||
next := ix.next(cur, f)
|
||||
next := ix.next(cur, f, digests)
|
||||
if next != nil {
|
||||
if err := ix.refuseDisagreement(*next); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case next == nil && cur == nil, next != nil && cur != nil && delivery.SameCandidate(*cur, *next):
|
||||
return nil, false, nil
|
||||
@@ -181,7 +201,12 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte,
|
||||
}
|
||||
|
||||
// next is the group after the message; nil when the group is left with no pull request.
|
||||
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Candidate {
|
||||
//
|
||||
// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of
|
||||
// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a
|
||||
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
|
||||
// directory.
|
||||
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate {
|
||||
var pulls []delivery.CandidatePull
|
||||
var base uint64
|
||||
if cur != nil {
|
||||
@@ -198,13 +223,20 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can
|
||||
switch {
|
||||
case f.DirsKnown:
|
||||
for _, dir := range f.Dirs {
|
||||
if m := ix.Builds.ModuleOf(f.Repository, dir); !slices.Contains(entry.Modules, m) {
|
||||
m := ix.Builds.ModuleOf(f.Repository, dir)
|
||||
if !slices.Contains(entry.Modules, m) {
|
||||
entry.Modules = append(entry.Modules, m)
|
||||
}
|
||||
if digest, ok := digests[dir]; ok {
|
||||
if entry.Fps == nil {
|
||||
entry.Fps = map[string]delivery.CandidateModule{}
|
||||
}
|
||||
entry.Fps[m] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview}
|
||||
}
|
||||
}
|
||||
case at >= 0 && pulls[at].Head.Head == f.Head:
|
||||
// Its directories are not known now, for the head the group already holds: what it knew stands.
|
||||
entry.Modules = pulls[at].Modules
|
||||
entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps
|
||||
}
|
||||
if at >= 0 {
|
||||
pulls[at] = entry
|
||||
@@ -226,5 +258,32 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can
|
||||
heads[i] = p.Head
|
||||
}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls}
|
||||
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
|
||||
}
|
||||
|
||||
// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
|
||||
// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445).
|
||||
const NoneNoPreview = "no merge preview is computed"
|
||||
|
||||
// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the
|
||||
// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is
|
||||
// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull
|
||||
// request is judged again.
|
||||
func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error {
|
||||
first := map[string]delivery.CandidatePull{}
|
||||
for _, p := range c.Pulls {
|
||||
for module, m := range p.Fps {
|
||||
before, ok := first[module]
|
||||
if !ok {
|
||||
first[module] = p
|
||||
continue
|
||||
}
|
||||
if before.Fps[module] != m {
|
||||
err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written",
|
||||
ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ func (m *memStore) Delete(_ context.Context, key string, rev uint64) error {
|
||||
// the controller's encoder; copied here, the vendored SDK does not carry testdata) has it, with the repository, module and path changed.
|
||||
func buildRecord(t *testing.T, repo, module, path, at string) []byte {
|
||||
t.Helper()
|
||||
raw := []byte(`{"id":"build-1792395447088936520","state":"built","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}`)
|
||||
raw := []byte(`{"id":"build-1792395447088936520","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}`)
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -306,11 +306,19 @@ func TestAConflictReadsAgain(t *testing.T) { // 1238 (compare-and-set: Update wi
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnusableBranchNameIsUnreadable(t *testing.T) {
|
||||
ix, _, _ := newIndexer(t)
|
||||
err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat x?", h1, "modules/a"))
|
||||
if err == nil || !strings.Contains(err.Error(), "unreadable") {
|
||||
t.Errorf("err = %v", err)
|
||||
// The SDK escapes a branch name's odd characters into a key (mesh-sdk a6dd698), so a name such as "feat x?" keys and is
|
||||
// indexed; no branch name is unreadable any more.
|
||||
func TestAnOddBranchNameKeysEscaped(t *testing.T) {
|
||||
ix, st, _ := newIndexer(t)
|
||||
if err := ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat x?", h1, "modules/a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, err := delivery.CandidatesKey("feat x?")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := st.vals[key]; !ok {
|
||||
t.Errorf("no candidate under %q", key)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The reasons a module has no fingerprint of the new form while the indexer writes none for any module (novox/hq
|
||||
// ADR 0465 clause 2; issue 1444). Toolchains missing is delivery.ToolchainNoReport.
|
||||
const (
|
||||
// None2NoManifest is for a module whose directory the merged message carries no manifest of (an older bridge).
|
||||
None2NoManifest = "the message carries no manifest"
|
||||
// None2Builder is for a module with no artifact of a toolchain: only the builder's fingerprint is missing.
|
||||
None2Builder = "builder's fingerprint not yet known"
|
||||
// None2Context is for a module an artifact of which reads another repository's tree (today's form gives it none
|
||||
// either, NoneContext).
|
||||
None2Context = "context not pinned"
|
||||
)
|
||||
|
||||
// lockFile is the lock file of a language's artifact that resolves packages when it is built, by name in the module's
|
||||
// directory (the bridge reads these, internal/bridge LockFiles; issue 1450).
|
||||
var lockFile = map[string]string{"go": "go.sum", "typescript": "package-lock.json"}
|
||||
|
||||
// moduleInputs is what a module's manifest and lock hashes give the indexer at a merge (issue 1450).
|
||||
type moduleInputs struct {
|
||||
// Kept is the new-form inputs the merge gave the module; nil when the manifest could not be read or gave none.
|
||||
Kept *delivery.KeptInputs2
|
||||
// Languages is the language of each artifact that has a toolchain (a bundle or a package of go or typescript), by
|
||||
// artifact name. The toolchain versions are joined to them when the build modules report them (issue 1444, the
|
||||
// watching half); the SDK's KeptInputs2 keeps no languages, so they are read from the manifest again then.
|
||||
Languages map[string]string
|
||||
// None2 is why the module has no fingerprint of the new form.
|
||||
None2 string
|
||||
}
|
||||
|
||||
// baseFP answers a base module's new-form fingerprint, or why it has none.
|
||||
type baseFP func(module string) (fp, none string)
|
||||
|
||||
// inputsOf turns the manifest of a module (raw, as the forge answered it), the tree of its directory and the lock
|
||||
// hashes of that directory (lock file name to hash) into its new-form inputs. The manifest is read by the catalogue's
|
||||
// parser; a manifest it refuses, or one that is of another module, leaves Kept nil and says why.
|
||||
//
|
||||
// The rules are the controller's builder (mesh-controller internal/builder/builder.go): a bundle or a package of a
|
||||
// language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages,
|
||||
// resolves packages when it is built (wantsPackages, :1017) only when its language's lock file exists: it then has
|
||||
// an entry in Locks with that file's hash, and an artifact whose lock file the merge did not carry has no entry (a
|
||||
// dependency-free or vendored bundle resolves nothing, as mesh-sdk SourceInputs2 says of an artifact not in Locks;
|
||||
// novox/hq ADR 0469 clause 5, issue 1459). A bundle that needs a lock and lacks one is refused where it is built, not
|
||||
// here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve
|
||||
// nothing and have no toolchain. A base is a build.on entry that names a module.
|
||||
func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
return moduleInputs{None2: "the manifest cannot be read: " + err.Error()}
|
||||
}
|
||||
if m.Module != module {
|
||||
return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)}
|
||||
}
|
||||
if tree == "" {
|
||||
return moduleInputs{None2: "the merge names no tree for the module's directory"}
|
||||
}
|
||||
kept := delivery.KeptInputs2{Tree: tree}
|
||||
contexts := false
|
||||
var missing []string
|
||||
langs := languagesOf(m)
|
||||
if m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Context != nil {
|
||||
contexts = true
|
||||
}
|
||||
}
|
||||
for _, name := range slices.Sorted(maps.Keys(langs)) {
|
||||
// A lock file the merge did not carry leaves no entry: the artifact resolves nothing (ADR 0469 clause 5).
|
||||
if h := locks[lockFile[langs[name]]]; h != "" {
|
||||
if kept.Locks == nil {
|
||||
kept.Locks = map[string]string{}
|
||||
}
|
||||
kept.Locks[name] = h
|
||||
}
|
||||
}
|
||||
for _, on := range basesOf(m) {
|
||||
if fp, none := base(on); fp != "" {
|
||||
kept.Bases = append(kept.Bases, fp)
|
||||
} else {
|
||||
missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", on, none))
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(kept.Bases)
|
||||
sort.Strings(missing)
|
||||
out := moduleInputs{Kept: &kept}
|
||||
if len(langs) > 0 {
|
||||
out.Languages = langs
|
||||
}
|
||||
switch {
|
||||
case contexts:
|
||||
out.None2 = None2Context
|
||||
case len(missing) > 0:
|
||||
out.None2 = strings.Join(missing, "; ")
|
||||
case len(langs) > 0:
|
||||
out.None2 = delivery.ToolchainNoReport
|
||||
default:
|
||||
out.None2 = None2Builder
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// languagesOf is the language of each artifact of the manifest that has a toolchain (a bundle or a package of go or
|
||||
// typescript), by artifact name: the artifacts a build module's toolchain enters the fingerprint of.
|
||||
func languagesOf(m catalogue.Manifest) map[string]string {
|
||||
langs := map[string]string{}
|
||||
if m.Build == nil {
|
||||
return langs
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactPackage {
|
||||
continue
|
||||
}
|
||||
if _, ok := lockFile[a.Language]; ok {
|
||||
langs[a.Name] = a.Language
|
||||
}
|
||||
}
|
||||
return langs
|
||||
}
|
||||
|
||||
// basesOf is the modules the manifest names as bases (its build.on entries that name a module), each once, sorted.
|
||||
func basesOf(m catalogue.Manifest) []string {
|
||||
if m.Build == nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, on := range m.Build.On {
|
||||
if on.Module != "" && !seen[on.Module] {
|
||||
seen[on.Module] = true
|
||||
out = append(out, on.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// aBase is the new-form fingerprint of a base module, the one SourceFingerprint2 gives for the inputs of avahi at a tree.
|
||||
func aBase(t *testing.T) string {
|
||||
t.Helper()
|
||||
fp, none := delivery.SourceFingerprint2(delivery.SourceInputs2{Module: "mesh-tools", Tree: "basetree"})
|
||||
if none != "" {
|
||||
t.Fatal(none)
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
func noBases(string) (string, string) { return "", "none are held in this test" }
|
||||
|
||||
func TestAGoModuleKeepsItsTreeAndTheHashOfItsLockFile(t *testing.T) { // 1450/AC1
|
||||
in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, noBases)
|
||||
want := &delivery.KeptInputs2{Tree: "avahitree", Locks: map[string]string{"tools": avahiGoSum}}
|
||||
if !reflect.DeepEqual(in.Kept, want) {
|
||||
t.Errorf("kept = %+v, want %+v", in.Kept, want)
|
||||
}
|
||||
if !reflect.DeepEqual(in.Languages, map[string]string{"tools": "go"}) {
|
||||
t.Errorf("languages = %v, want tools: go", in.Languages)
|
||||
}
|
||||
// Through the indexer: the merge's tree of the module's directory and the lock hashes of that directory, kept in modules.avahi.
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "avahitree", "modules/other": "x"}),
|
||||
map[string]string{"modules/avahi": catalogueManifest(t, "avahi")}, map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "avahi")
|
||||
if !ok || !reflect.DeepEqual(m.Inputs2, want) || m.Manifest != delivery.ManifestDigest([]byte(catalogueManifest(t, "avahi"))) {
|
||||
t.Errorf("module = %+v (%v), want inputs %+v and the manifest digest", m, ok, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtifactWhoseLockFileTheMergeDidNotCarryHasNoLocksEntry(t *testing.T) { // 1459/AC1, AC3
|
||||
// A Go bundle without a go.sum (a dependency-free or vendored module): no entry, so a fingerprint is possible (ADR 0469 clause 5).
|
||||
in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", nil, noBases)
|
||||
if in.Kept == nil || len(in.Kept.Locks) != 0 {
|
||||
t.Fatalf("kept = %+v, want no Locks entry", in.Kept)
|
||||
}
|
||||
if !reflect.DeepEqual(in.Languages, map[string]string{"tools": "go"}) {
|
||||
t.Errorf("languages = %v, want tools: go (the toolchain still applies)", in.Languages)
|
||||
}
|
||||
// With the toolchain known, mesh-sdk gives such inputs a src2: value, never 'no lock'.
|
||||
if fp, none := delivery.SourceFingerprint2(in.Kept.With("avahi", map[string]delivery.Toolchain2{"tools": {Language: "go", Version: "1.25.0", Recipe: "1"}})); fp == "" || none != "" {
|
||||
t.Errorf("fingerprint = %q none %q, want a value", fp, none)
|
||||
}
|
||||
// A TypeScript bundle without package-lock.json (go.sum of another language does not count).
|
||||
ts := inputsOf("audit-logger", catalogueManifest(t, "audit-logger"), "t", map[string]string{"go.sum": "wrong"}, noBases)
|
||||
if ts.Kept == nil || len(ts.Kept.Locks) != 0 || !reflect.DeepEqual(ts.Languages, map[string]string{"code": "typescript"}) {
|
||||
t.Errorf("typescript: kept %+v languages %v", ts.Kept, ts.Languages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtifactWhoseLockFileTheMergeCarriedHasItsHash(t *testing.T) { // 1459/AC2, AC3
|
||||
ts := inputsOf("audit-logger", catalogueManifest(t, "audit-logger"), "t", map[string]string{"go.sum": "wrong", "package-lock.json": avahiGoSum}, noBases)
|
||||
if ts.Kept == nil || !reflect.DeepEqual(ts.Kept.Locks, map[string]string{"code": avahiGoSum}) {
|
||||
t.Errorf("typescript: kept %+v", ts.Kept)
|
||||
}
|
||||
// netcheck has a Go and a TypeScript bundle: each takes its own language's lock file, and one without it has none.
|
||||
both := inputsOf("netcheck", catalogueManifest(t, "netcheck"), "t", map[string]string{"go.sum": "g", "package-lock.json": "p"}, noBases)
|
||||
if both.Kept == nil || !reflect.DeepEqual(both.Kept.Locks, map[string]string{"tools-go": "g", "tools-typescript": "p"}) {
|
||||
t.Errorf("netcheck: kept %+v", both.Kept)
|
||||
}
|
||||
goOnly := inputsOf("netcheck", catalogueManifest(t, "netcheck"), "t", map[string]string{"go.sum": "g"}, noBases)
|
||||
if goOnly.Kept == nil || !reflect.DeepEqual(goOnly.Kept.Locks, map[string]string{"tools-go": "g"}) {
|
||||
t.Errorf("netcheck without package-lock.json: kept %+v", goOnly.Kept)
|
||||
}
|
||||
}
|
||||
|
||||
// aModuleWithABase is avahi's manifest (mesh-catalog a67f0196) with the one build.on entry that names a module as the
|
||||
// controller's own test has it (mesh-controller internal/builder/standing_on_test.go:22): no manifest of the sources names
|
||||
// a module as a base, only vendors' images.
|
||||
func aModuleWithABase(t *testing.T) string {
|
||||
t.Helper()
|
||||
raw := catalogueManifest(t, "avahi")
|
||||
with := strings.Replace(raw, `"build": {`, `"build": {
|
||||
"on": [{"arg": "RUNTIME_BASE", "module": "mesh-tools", "artifact": "runtime"}],`, 1)
|
||||
if with == raw {
|
||||
t.Fatal("avahi's manifest has no build")
|
||||
}
|
||||
return with
|
||||
}
|
||||
|
||||
func TestABaseEntersByTheFingerprintOfItsModuleOrNamesItsAbsence(t *testing.T) { // 1450/AC3
|
||||
fp := aBase(t)
|
||||
has := func(module string) (string, string) {
|
||||
if module != "mesh-tools" {
|
||||
t.Errorf("asked for the base %q", module)
|
||||
}
|
||||
return fp, ""
|
||||
}
|
||||
in := inputsOf("avahi", aModuleWithABase(t), "t", map[string]string{"go.sum": avahiGoSum}, has)
|
||||
if in.Kept == nil || !reflect.DeepEqual(in.Kept.Bases, []string{fp}) {
|
||||
t.Errorf("bases = %+v, want [%s]", in.Kept, fp)
|
||||
}
|
||||
none := inputsOf("avahi", aModuleWithABase(t), "t", map[string]string{"go.sum": avahiGoSum}, noBases)
|
||||
if none.Kept == nil || len(none.Kept.Bases) != 0 || !strings.Contains(none.None2, "mesh-tools") || !strings.Contains(none.None2, "base") {
|
||||
t.Errorf("kept %+v none2 %q, want no bases and none2 naming the base mesh-tools", none.Kept, none.None2)
|
||||
}
|
||||
// Through the indexer: modules.mesh-tools holds a fingerprint of the new form (a record written by a later release).
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
mods.put("mesh-tools", mustEncode(t, delivery.Module{Repository: catalogueRepo, Commit: merge, None: "x", FP2: fp, Manifest: strings.Repeat("a", 64)}))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t"}), map[string]string{"modules/avahi": aModuleWithABase(t)}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.Inputs2 == nil || !reflect.DeepEqual(m.Inputs2.Bases, []string{fp}) {
|
||||
t.Errorf("module = %+v, want the base's fingerprint", m)
|
||||
}
|
||||
}
|
||||
|
||||
func mustEncode(t *testing.T, m delivery.Module) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeModule(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestAnArtifactOfNoToolchainHasNoLanguageAndNoLock(t *testing.T) { // 1450/AC4
|
||||
in := inputsOf("ca-trust", catalogueManifest(t, "ca-trust"), "t", map[string]string{"go.sum": avahiGoSum}, noBases)
|
||||
if in.Kept == nil || len(in.Kept.Locks) != 0 || len(in.Languages) != 0 {
|
||||
t.Errorf("kept %+v languages %v, want neither locks nor languages for an archive", in.Kept, in.Languages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAManifestTheParserRefusesLeavesNoInputsAndSaysWhy(t *testing.T) { // 1450/AC5
|
||||
for name, raw := range map[string]string{
|
||||
"not json": "not json",
|
||||
"unknown field": strings.Replace(catalogueManifest(t, "avahi"), `"version": "1",`, `"version": "1", "scope": "everything",`, 1),
|
||||
} {
|
||||
in := inputsOf("avahi", raw, "t", nil, noBases)
|
||||
if in.Kept != nil || !strings.HasPrefix(in.None2, "the manifest cannot be read: ") || len(in.None2) <= len("the manifest cannot be read: ") {
|
||||
t.Errorf("%s: kept %+v none2 %q, want nil and 'the manifest cannot be read: <error>'", name, in.Kept, in.None2)
|
||||
}
|
||||
}
|
||||
if in := inputsOf("someone-else", catalogueManifest(t, "avahi"), "t", nil, noBases); in.Kept != nil || !strings.Contains(in.None2, "avahi") {
|
||||
t.Errorf("a manifest of another module: kept %+v none2 %q", in.Kept, in.None2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAContextIsNotPinned(t *testing.T) { // 1450 (the rule 1444 keeps: context not pinned, as today's form)
|
||||
// build-agent's manifest from mesh-controller at f2d7bc94 has an image artifact built from a context in another repository.
|
||||
in := inputsOf("build-agent", fixture(t, "controller-f2d7bc94-build-agent"), "t", nil, noBases)
|
||||
if in.Kept == nil || in.None2 != None2Context {
|
||||
t.Errorf("kept %+v none2 %q, want none2 %q", in.Kept, in.None2, None2Context)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The build records below are the live ones, as `mesh-controller.command builds` showed them on 2026-10-19 at about
|
||||
// 21:20 local (the last 100 builds): id, state, module, repository (the clone URL, as the record names it), commit (the
|
||||
// eight characters the listing shows), the time (the listing's local minute, as UTC), the source fingerprint, and the
|
||||
// ref only where the listing shows "at main"; a build a delivery plan asked after a merge shows none, so the record
|
||||
// carries none. No reading command shows a whole record, so two fields come from elsewhere: the path is the module's
|
||||
// directory in the catalogue (a clone of mesh-catalog and mesh-media-catalog, origin/main at the same time), and the
|
||||
// source-inputs are mailu's live inputs (build-1792391134620131370, captured in mesh-sdk go/delivery/fingerprint_test.go
|
||||
// TestFingerprintLiveBuilds) with the module's name put in; the indexer reads only their bases, contexts and toolchains.
|
||||
const (
|
||||
liveTree = "cfc6fa8ff5fc1e220132e51ade653b9d3f387b3e"
|
||||
liveInputsTail = `"Tree":"` + liveTree + `","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,` +
|
||||
`"Toolchains":{"code":"typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"},"Unpinned":""}`
|
||||
catalogURL = "http://novox.internal:20000/novox/mesh-catalog.git"
|
||||
mediaCatalogURL = "http://novox.internal:20000/novox/mesh-media-catalog.git"
|
||||
)
|
||||
|
||||
func liveRecord(id, module, repository, ref, commit, path, at, source string) [2]any {
|
||||
raw := `{"id":"` + id + `","repository":"` + repository + `",`
|
||||
if ref != "" {
|
||||
raw += `"ref":"` + ref + `",`
|
||||
}
|
||||
raw += `"module":"` + module + `","commit":"` + commit + `","path":"` + path + `","at":"` + at + `","source-fingerprint":"` + source +
|
||||
`","source-inputs":{"Module":"` + module + `",` + liveInputsTail + `}`
|
||||
return [2]any{delivery.BuildKey(module, id), []byte(raw)}
|
||||
}
|
||||
|
||||
func liveBuilds() [][2]any {
|
||||
return [][2]any{
|
||||
// a plan-asked build after a merge: no ref
|
||||
liveRecord("build-1792417485811374131", "zsh", catalogURL, "", "e8c34487", "modules/zsh", "2026-10-19T13:57:00Z", "src1:7e600dde59f3d7f019af00741583304288883e7d3ed955f4c61da26474f3be63"),
|
||||
liveRecord("build-1792435539606203626", "mesh-delivery", catalogURL, "", "9efdd87a", "modules/mesh-delivery", "2026-10-19T18:45:00Z", "src1:f79e2eedc2689cf6b3591ebfdb81ec65899131122fb2fe3dd9e384091d8a2a6d"),
|
||||
// a build at main
|
||||
liveRecord("build-1792417463192745880", "mailu", catalogURL, "main", "e8c34487", "modules/mailu", "2026-10-19T13:50:00Z", "src1:98d35b17d3a3295e3a0efed2b8c4316c9a9b32ec155f8343824654661b7383a2"),
|
||||
// a module of another repository
|
||||
liveRecord("build-1792417481951691634", "sonarr", mediaCatalogURL, "main", "197abb06", "modules/sonarr", "2026-10-19T13:57:00Z", "src1:0fff753b8c66aa8a8ac1f38d210cea9512a78a8592dd2475bfe6ade77ff21b61"),
|
||||
}
|
||||
}
|
||||
|
||||
func liveIndexer(t *testing.T) (*Indexer, *memStore) {
|
||||
t.Helper()
|
||||
ix, mods := modulesIndexer(t, liveBuilds()...)
|
||||
return ix, mods
|
||||
}
|
||||
|
||||
func TestMergeWritesEveryModuleTheLiveRecordsNameInTheRepository(t *testing.T) { // 1281/AC3
|
||||
ix, mods := liveIndexer(t)
|
||||
msg := mergedMsg("novox/mesh-catalog", map[string]string{
|
||||
"modules/zsh": liveTree, "modules/mesh-delivery": "newtree", "modules/mailu": "newtree", "modules/sonarr": "newtree",
|
||||
})
|
||||
if err := ix.Apply(context.Background(), msg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"zsh", "mesh-delivery", "mailu"} {
|
||||
m, ok := moduleOf(t, mods, name)
|
||||
if !ok {
|
||||
t.Errorf("%s: no module key written", name)
|
||||
continue
|
||||
}
|
||||
if (m.FP == "") == (m.None == "") || m.Repository != "novox/mesh-catalog" || m.Commit != merge {
|
||||
t.Errorf("%s: module = %+v, want an fp or a none, repository novox/mesh-catalog, commit %s", name, m, merge)
|
||||
}
|
||||
}
|
||||
// the tree of zsh at the merge is the tree its inputs record, so its fingerprint is the one those inputs give
|
||||
var in delivery.SourceInputs
|
||||
in.Module, in.Tree = "zsh", liveTree
|
||||
in.Bases = []string{"sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"}
|
||||
in.Toolchains = map[string]string{"code": "typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"}
|
||||
want, _ := delivery.SourceFingerprint(in)
|
||||
if m, _ := moduleOf(t, mods, "zsh"); m.FP != want {
|
||||
t.Errorf("zsh fp = %q, want %q", m.FP, want)
|
||||
}
|
||||
if _, ok := mods.vals["sonarr"]; ok {
|
||||
t.Error("the key of a module of mesh-media-catalog was written at a merge of mesh-catalog")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChangedDirectoryMapsToTheModuleItsLiveRecordNamesByPath(t *testing.T) { // 1281/AC4
|
||||
ix, _ := liveIndexer(t)
|
||||
for dir, want := range map[string]string{"modules/zsh": "zsh", "modules/mesh-delivery": "mesh-delivery", "modules/mailu/": "mailu"} {
|
||||
if got, ok := ix.Builds.Module("novox/mesh-catalog", dir); !ok || got != want {
|
||||
t.Errorf("Module(novox/mesh-catalog, %q) = %q, %v; want %q", dir, got, ok, want)
|
||||
}
|
||||
}
|
||||
if got, ok := ix.Builds.Module("novox/mesh-catalog", "modules/sonarr"); ok {
|
||||
t.Errorf("Module found %q for a directory of mesh-media-catalog under mesh-catalog", got)
|
||||
}
|
||||
if got := ix.Builds.ModuleOf("novox/mesh-media-catalog", "modules/sonarr"); got != "sonarr" {
|
||||
t.Errorf("ModuleOf(mesh-media-catalog, modules/sonarr) = %q, want sonarr", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests
|
||||
// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the
|
||||
// module a build record names the directory the path of.
|
||||
func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string {
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
}
|
||||
if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" {
|
||||
return m.Module
|
||||
}
|
||||
name, _ := ix.Builds.Module(repository, dir)
|
||||
return name
|
||||
}
|
||||
|
||||
// keepManifest writes manifests.<digest> for a manifest as the forge answered it, create-once (issue 1449): a key that
|
||||
// exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for
|
||||
// one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the
|
||||
// digest. A manifest that cannot be a value (empty, not UTF-8, not JSON) is an error the caller says; it is never tried
|
||||
// again. With no manifests store nothing is written and the digest is still answered.
|
||||
func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string, error) {
|
||||
digest := delivery.ManifestDigest([]byte(raw))
|
||||
if ix.Manifests == nil {
|
||||
return digest, nil
|
||||
}
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: manifests.%s: %v", errCannotBeAValue, key, err)
|
||||
}
|
||||
err = ix.Manifests.Create(ctx, key, data)
|
||||
if err == nil {
|
||||
return digest, nil
|
||||
}
|
||||
if !errors.Is(err, ErrConflict) {
|
||||
return "", fmt.Errorf("writing manifests.%s: %w", key, err)
|
||||
}
|
||||
held, _, found, err := ix.Manifests.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading manifests.%s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return "", fmt.Errorf("manifests.%s was there and is not: written again at the next delivery", key)
|
||||
}
|
||||
if cur, err := delivery.DecodeManifest(held); err == nil && delivery.SameManifest(cur, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw}) {
|
||||
return digest, nil
|
||||
}
|
||||
return "", fmt.Errorf("manifests.%s holds another value than the manifest of %s, which hashes to it", key, module)
|
||||
}
|
||||
|
||||
// keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory.
|
||||
// A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest.
|
||||
func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) {
|
||||
digests := make(map[string]string, len(f.Manifests))
|
||||
for dir, raw := range f.Manifests {
|
||||
module := ix.moduleNameOf(f.Repository, dir, raw)
|
||||
digest, err := ix.keepManifest(ctx, module, raw)
|
||||
switch {
|
||||
case err == nil:
|
||||
digests[dir] = digest
|
||||
case errors.Is(err, errCannotBeAValue):
|
||||
ix.logf("%s#%d: the manifest of %q cannot be kept: %v", f.Repository, f.Number, dir, err)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return digests, nil
|
||||
}
|
||||
|
||||
var errCannotBeAValue = errors.New("not a manifest value")
|
||||
@@ -0,0 +1,102 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The manifests these tests carry are real files, copied unchanged into testdata: the module.json of modules of
|
||||
// mesh-catalog at its main commit a67f01963f0d9ef0913a63720b89e573ca7a16ef (testdata/catalogue-a67f0196-<module>.module.json)
|
||||
// and of build-agent from mesh-controller at its main commit f2d7bc94 (testdata/controller-f2d7bc94-build-agent.module.json).
|
||||
func fixture(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile("testdata/" + name + ".module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func catalogueManifest(t *testing.T, module string) string {
|
||||
return fixture(t, "catalogue-a67f0196-"+module)
|
||||
}
|
||||
|
||||
// avahiGoSum is the sha256 of modules/avahi/go.sum in mesh-catalog at a67f0196 (sha256sum of that file).
|
||||
const avahiGoSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||
|
||||
const catalogueRepo = "novox/mesh-catalog"
|
||||
|
||||
func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memStore) {
|
||||
t.Helper()
|
||||
ix, mods := modulesIndexer(t, recs...)
|
||||
manifests := newMem()
|
||||
ix.Manifests = manifests
|
||||
return ix, mods, manifests
|
||||
}
|
||||
|
||||
func withManifests(f delivery.Forge, manifests map[string]string, locks map[string]map[string]string) delivery.Forge {
|
||||
f.Manifests, f.Locks = manifests, locks
|
||||
return f
|
||||
}
|
||||
|
||||
func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := delivery.ManifestDigest([]byte(avahi))
|
||||
raw, ok := manifests.vals[digest]
|
||||
if !ok || len(manifests.vals) != 1 {
|
||||
t.Fatalf("manifests holds %d keys, want only %s", len(manifests.vals), digest)
|
||||
}
|
||||
m, err := delivery.DecodeManifest(raw)
|
||||
if err != nil || m.Module != "avahi" || m.JSON != avahi {
|
||||
t.Errorf("manifest = %+v err %v, want avahi and the bytes of the file", m, err)
|
||||
}
|
||||
if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != digest {
|
||||
t.Errorf("module = %+v (%v), want the manifest digest %s", mod, ok, digest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, _, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A second merge, and a pull message carrying it, the digest being the same.
|
||||
f.Merge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Errorf("a second merge: %v", err)
|
||||
}
|
||||
pull := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
|
||||
if err := ix.Apply(context.Background(), pull); err != nil {
|
||||
t.Errorf("a pull message: %v", err)
|
||||
}
|
||||
if manifests.writes != 1 || len(manifests.vals) != 1 {
|
||||
t.Errorf("manifests written %d times into %d keys, want once into one", manifests.writes, len(manifests.vals))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a manifest that is not JSON cannot be kept byte for byte)
|
||||
var logs []string
|
||||
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": "not json"}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(manifests.vals) != 0 || len(logs) == 0 {
|
||||
t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs)
|
||||
}
|
||||
if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" {
|
||||
t.Errorf("module = %+v (%v), want no manifest digest", mod, ok)
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -45,8 +44,18 @@ func fingerprintOf(k Known, trees map[string]string) (fp, none string) {
|
||||
}
|
||||
|
||||
// applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and
|
||||
// why). A module of another repository is not written. A value the module already holds is not written again.
|
||||
// why), with the source revision that names the merge commit as the repository's head when the indexer has seen it
|
||||
// (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already
|
||||
// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it
|
||||
// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449,
|
||||
// 1450); the fingerprint itself is computed by Recompute (toolchains.go) once the merge is written, and again when a build
|
||||
// module reports its toolchain.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
digests, err := ix.keepManifests(ctx, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
overlay := map[string]delivery.Module{}
|
||||
for _, k := range ix.Builds.Modules(f.Repository) {
|
||||
key, err := delivery.ModulesKey(k.Module)
|
||||
if err != nil {
|
||||
@@ -54,21 +63,58 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
continue
|
||||
}
|
||||
fp, none := fingerprintOf(k, f.Trees)
|
||||
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none}
|
||||
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && delivery.SameModule(cur, withV(next)) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
data, err := delivery.EncodeModule(next)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path], Path: pathOf(k.Path)}
|
||||
// The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which
|
||||
// names what is missing, from its manifest and lock hashes (inputs.go).
|
||||
raw, has := f.Manifests[k.Path]
|
||||
switch {
|
||||
case !has:
|
||||
next.None2 = None2NoManifest
|
||||
case next.Manifest == "":
|
||||
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
|
||||
default:
|
||||
in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx))
|
||||
next.Inputs2, next.None2 = in.Kept, in.None2
|
||||
}
|
||||
overlay[key] = next
|
||||
}
|
||||
return nil
|
||||
// What this merge moved reaches the modules standing on it (a base's tree, build-agent's fingerprint), in the graph's order.
|
||||
return ix.recomputeWith(ctx, overlay)
|
||||
}
|
||||
|
||||
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
|
||||
func (ix *Indexer) baseFP(ctx context.Context) baseFP {
|
||||
return func(module string) (string, string) {
|
||||
key, err := delivery.ModulesKey(module)
|
||||
if err != nil {
|
||||
return "", err.Error()
|
||||
}
|
||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", "its record cannot be read: " + err.Error()
|
||||
}
|
||||
if !found {
|
||||
return "", "the indexer holds no record of it"
|
||||
}
|
||||
cur, err := delivery.DecodeModule(value)
|
||||
switch {
|
||||
case err != nil:
|
||||
return "", "its record cannot be read: " + err.Error()
|
||||
case cur.FP2 != "":
|
||||
return cur.FP2, ""
|
||||
case cur.None2 != "":
|
||||
return "", cur.None2
|
||||
}
|
||||
return "", "its record carries no fingerprint of the new form"
|
||||
}
|
||||
}
|
||||
|
||||
// pathOf is the module's directory as Module.Path holds it: "." for the repository's root, since empty means not known.
|
||||
func pathOf(dir string) string {
|
||||
if dir == "" {
|
||||
return delivery.RootPath
|
||||
}
|
||||
return dir
|
||||
}
|
||||
|
||||
func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }
|
||||
|
||||
@@ -15,7 +15,16 @@ const merge = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
// the captured shape): SourceInputs under its Go field names, no json keys. inputs is that object's JSON.
|
||||
func record(t *testing.T, id, module, repo, path, ref, state, at, inputs string) (string, []byte) {
|
||||
t.Helper()
|
||||
raw := `{"id":"` + id + `","state":"` + state + `","repository":"` + repo + `","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"` + path + `","at":"` + at + `"`
|
||||
// an outcome has no state key (mesh-controller internal/chain/records.go:57-59); state asked is a request
|
||||
raw := `{"id":"` + id + `",`
|
||||
switch state {
|
||||
case "":
|
||||
case "failed": // a failure is an outcome with failed set and no state (mesh-controller internal/chain/records.go:109)
|
||||
raw += `"failed":"the build failed",`
|
||||
default:
|
||||
raw += `"state":"` + state + `",`
|
||||
}
|
||||
raw += `"repository":"` + repo + `","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"` + path + `","at":"` + at + `"`
|
||||
if inputs != "" {
|
||||
raw += `,"source-fingerprint":"src1:ignored","source-inputs":` + inputs
|
||||
}
|
||||
@@ -62,7 +71,7 @@ func moduleOf(t *testing.T, st *memStore, key string) (delivery.Module, bool) {
|
||||
const aInputs = `{"Module":"a","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||
|
||||
func TestModuleFingerprintFromTheMergeTreeAndTheRecordedInputs(t *testing.T) { // 1239/AC1
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs))
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree", "modules/b": "other"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -88,7 +97,7 @@ func TestMergeOfTheRecordedTreeReproducesTheRecordedFingerprint(t *testing.T) {
|
||||
const recorded = "src1:539f2e64000d436c572ef53b40e524012455945f8c5a1c6bb173ee0707e88ef4"
|
||||
inputs := `{"Module":"mailu","Tree":"` + tree + `","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,` +
|
||||
`"Toolchains":{"code":"typescript sha256:4b81682df5665c1f461d3f720a3a0fd7593f4aa71e4fc272dbfdc4bbd235b489 20125bd590041690"},"Unpinned":""}`
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1792391134620131370", "mailu", "novox/mesh-catalog", "modules/mailu", "main", "built", "2026-10-19T09:00:00Z", inputs))
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1792391134620131370", "mailu", "novox/mesh-catalog", "modules/mailu", "main", "", "2026-10-19T09:00:00Z", inputs))
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-catalog", map[string]string{"modules/mailu": tree})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -101,7 +110,7 @@ func TestANarrowContextHasNoFingerprint(t *testing.T) { // 1239/AC3, 1209/AC1
|
||||
// A Go program's build source as the builder records it (build-agent's record, mesh-sdk go/delivery/testdata):
|
||||
// the context "server" is "narrow:" and a hash.
|
||||
inputs := `{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""}`
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "build-agent", "novox/mesh-controller", "modules/build-agent", "main", "built", "2026-10-19T09:00:00Z", inputs))
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "build-agent", "novox/mesh-controller", "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", inputs))
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-controller", map[string]string{"modules/build-agent": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -113,7 +122,7 @@ func TestANarrowContextHasNoFingerprint(t *testing.T) { // 1239/AC3, 1209/AC1
|
||||
|
||||
func TestUnpinnedBuildHasNoFingerprint(t *testing.T) { // 1239/AC4
|
||||
inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":null,"Toolchains":null,"Unpinned":"the build resolves packages when built"}`
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", inputs))
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "the build resolves packages when built" {
|
||||
t.Errorf("module = %+v (%v)", m, ok)
|
||||
@@ -122,7 +131,7 @@ func TestUnpinnedBuildHasNoFingerprint(t *testing.T) { // 1239/AC4
|
||||
|
||||
func TestARootTreeContextHasNoFingerprint(t *testing.T) { // 1239/AC5
|
||||
inputs := `{"Module":"a","Tree":"t","Bases":null,"Contexts":{"web":"f00dcafef00dcafef00dcafef00dcafef00dcafe"},"Toolchains":null,"Unpinned":""}`
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", inputs))
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", inputs))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
if !ok || m.FP != "" || !strings.Contains(m.None, "root tree") || !strings.Contains(m.None, "release 4") {
|
||||
@@ -131,7 +140,7 @@ func TestARootTreeContextHasNoFingerprint(t *testing.T) { // 1239/AC5
|
||||
}
|
||||
|
||||
func TestDirectoryGoneFromTheMergeHasNoTree(t *testing.T) { // 1239/AC6
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs))
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/other": "t"}))
|
||||
if m, ok := moduleOf(t, mods, "a"); !ok || m.FP != "" || m.None != "no tree" {
|
||||
t.Errorf("module = %+v (%v)", m, ok)
|
||||
@@ -141,8 +150,8 @@ func TestDirectoryGoneFromTheMergeHasNoTree(t *testing.T) { // 1239/AC6
|
||||
func TestAModuleWithNoSuccessfulTrunkBuildHasNone(t *testing.T) { // 1239/AC7
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "failed-only", "novox/mesh-delivery", "modules/f", "main", "failed", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-2", "branch-only", "novox/mesh-delivery", "modules/g", "feat/x", "built", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-3", "no-inputs", "novox/mesh-delivery", "modules/h", "main", "built", "2026-10-19T09:00:00Z", ""),
|
||||
rec(t, "build-2", "branch-only", "novox/mesh-delivery", "modules/g", "feat/x", "", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-3", "no-inputs", "novox/mesh-delivery", "modules/h", "main", "", "2026-10-19T09:00:00Z", ""),
|
||||
)
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/f": "t", "modules/g": "t", "modules/h": "t"}))
|
||||
for _, name := range []string{"failed-only", "branch-only", "no-inputs"} {
|
||||
@@ -155,10 +164,10 @@ func TestAModuleWithNoSuccessfulTrunkBuildHasNone(t *testing.T) { // 1239/AC7
|
||||
func TestTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1239 (the newest key is only a hint)
|
||||
older := strings.Replace(aInputs, `"Bases":["registry.example/base@sha256:aaa"]`, `"Bases":["registry.example/base@sha256:old"]`, 1)
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", older),
|
||||
rec(t, "build-2", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T10:00:00Z", aInputs),
|
||||
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", older),
|
||||
rec(t, "build-2", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T10:00:00Z", aInputs),
|
||||
rec(t, "build-3", "a", "novox/mesh-delivery", "modules/a", "main", "failed", "2026-10-19T11:00:00Z", older),
|
||||
rec(t, "build-4", "a", "novox/mesh-delivery", "modules/a", "feat/x", "built", "2026-10-19T12:00:00Z", older),
|
||||
rec(t, "build-4", "a", "novox/mesh-delivery", "modules/a", "feat/x", "", "2026-10-19T12:00:00Z", older),
|
||||
)
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
want, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
|
||||
@@ -170,8 +179,8 @@ func TestTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1239 (the newest
|
||||
|
||||
func TestModulesOfOtherRepositoriesAreNotWritten(t *testing.T) { // 1239/AC8
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "built", "2026-10-19T09:00:00Z", strings.Replace(aInputs, `"Module":"a"`, `"Module":"c"`, 1)))
|
||||
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", strings.Replace(aInputs, `"Module":"a"`, `"Module":"c"`, 1)))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "t", "modules/c": "t"}))
|
||||
if _, ok := mods.vals["c"]; ok {
|
||||
t.Error("the key of a module of another repository was written")
|
||||
@@ -182,7 +191,7 @@ func TestModulesOfOtherRepositoriesAreNotWritten(t *testing.T) { // 1239/AC8
|
||||
}
|
||||
|
||||
func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value writes nothing)
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs))
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
m := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})
|
||||
_ = ix.Apply(context.Background(), m)
|
||||
rev, writes := mods.revs["a"], mods.writes
|
||||
@@ -193,7 +202,7 @@ func TestAnUnchangedModuleWritesNothing(t *testing.T) { // 1239 (unchanged value
|
||||
}
|
||||
|
||||
func TestMergedMessageRemovesThePullFromItsGroupToo(t *testing.T) { // 1238/AC5 with 1239 (both effects of one message)
|
||||
ix, _ := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "built", "2026-10-19T09:00:00Z", aInputs))
|
||||
ix, _ := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
cands := ix.Store.(*memStore)
|
||||
_ = ix.Apply(context.Background(), pull("novox/mesh-delivery", 12, "feat/x", h1, "modules/a"))
|
||||
_ = ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
@@ -201,3 +210,56 @@ func TestMergedMessageRemovesThePullFromItsGroupToo(t *testing.T) { // 1238/AC5
|
||||
t.Error("the merged pull request is still a candidate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARecordAsTheControllerWritesItGivesAFingerprint(t *testing.T) { // 1281/AC8
|
||||
// The layout of mesh-controller internal/chain/records.go Build (commit 1f708e2d8d72): an outcome has no state key, a build
|
||||
// asked by a plan after a merge no ref, and the repository is the clone URL.
|
||||
raw := `{"id":"build-1","repository":"http://novox.internal:20000/novox/mesh-delivery.git","module":"a","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20",` +
|
||||
`"path":"modules/a","source-fingerprint":"src1:ignored","source-inputs":` + aInputs + `,"at":"2026-10-19T09:00:00Z"}`
|
||||
ix, mods := modulesIndexer(t, [2]any{delivery.BuildKey("a", "build-1"), []byte(raw)})
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
if !ok || m.FP == "" || m.None != "" {
|
||||
t.Errorf("module = %+v (%v), want a fingerprint, not none", m, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeWritesTheModulesDirectoryAndKeepsItThroughSourceAndRecompute(t *testing.T) { // 1606/AC1, AC2
|
||||
ix, mods := modulesIndexer(t,
|
||||
rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-2", "r", "novox/mesh-delivery", "", "main", "", "2026-10-19T09:00:00Z", strings.Replace(aInputs, `"Module":"a"`, `"Module":"r"`, 1)))
|
||||
ctx := context.Background()
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree", "": "roottree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for module, want := range map[string]string{"a": "modules/a", "r": delivery.RootPath} {
|
||||
if m, ok := moduleOf(t, mods, module); !ok || m.Path != want {
|
||||
t.Errorf("after the merge %s = %+v (%v), want path %q", module, m, ok, want)
|
||||
}
|
||||
}
|
||||
if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.Recompute(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Path != "modules/a" || m.Src != 7 {
|
||||
t.Errorf("after the source and a recompute a = %+v, want path modules/a kept and Src 7", m)
|
||||
}
|
||||
raw := string(mods.vals["a"])
|
||||
if !strings.Contains(raw, `"path":"modules/a"`) {
|
||||
t.Errorf("the stored value %s has no path", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARootModuleHasTheDirectoryDot(t *testing.T) { // 1615/AC3: empty means not known, a root module is "."
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "r", "novox/mesh-sdk", "", "main", "", "2026-10-19T09:00:00Z", strings.Replace(aInputs, `"Module":"a"`, `"Module":"r"`, 1)))
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-sdk", map[string]string{"": "roottree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "r"); !ok || m.Path != delivery.RootPath {
|
||||
t.Errorf("module = %+v (%v), want path %q", m, ok, delivery.RootPath)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go).
|
||||
func wantAvahiFP(t *testing.T) string {
|
||||
t.Helper()
|
||||
fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
|
||||
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
|
||||
if none != "" {
|
||||
t.Fatal(none)
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z",
|
||||
`{"Module":"avahi","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
|
||||
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "avahi")
|
||||
if !ok {
|
||||
t.Fatal("no module avahi")
|
||||
}
|
||||
todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
|
||||
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
|
||||
if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum {
|
||||
t.Errorf("module = %+v, want its manifest digest and inputs", m)
|
||||
}
|
||||
if m.None2 != "toolchain not yet reported" || m.FP2 != "" {
|
||||
t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2)
|
||||
}
|
||||
if m.FP != todays || m.None != "" {
|
||||
t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) { // 1444/AC2
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "ca-trust", catalogueRepo, "modules/ca-trust", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/ca-trust": "newtree"}), map[string]string{"modules/ca-trust": catalogueManifest(t, "ca-trust")}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.None2 != "builder's fingerprint not yet known" || m.Inputs2 == nil {
|
||||
t.Errorf("module = %+v (%v), want none2 naming the builder's fingerprint", m, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3
|
||||
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" {
|
||||
t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok)
|
||||
}
|
||||
if len(manifests.vals) != 0 {
|
||||
t.Errorf("manifests holds %d keys", len(manifests.vals))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSameMergeAppliedAgainWritesNothing(t *testing.T) { // 1444 (a value the module already holds is not written again, with inputs kept)
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
|
||||
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mods.writes != 1 {
|
||||
t.Errorf("modules written %d times, want once", mods.writes)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The gate's source revision (novox/hq ADR 0437 §5, ADR 0450 §1): source.main names every repository's merged head, and its
|
||||
// revision in the bucket is the mesh's source revision. The indexer keeps the latest it has seen. A module is stamped
|
||||
// with it (Module.Src) when the head the source names for its repository is the merge commit the module's fingerprint
|
||||
// was computed at: from the modules bucket when the merge was heard first, and when the merge is heard after the source.
|
||||
|
||||
// sourceNow is the latest source revision seen.
|
||||
type sourceNow struct {
|
||||
rev uint64
|
||||
heads map[string]string
|
||||
}
|
||||
|
||||
// srcOf is the source revision that names this merge commit as the repository's head, or 0 when the latest source seen
|
||||
// names another head or none has been seen.
|
||||
func (ix *Indexer) srcOf(repository, merge string) uint64 {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
if ix.source.rev != 0 && merge != "" && ix.source.heads[repository] == merge {
|
||||
return ix.source.rev
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ApplySource takes a revision of the gate's source.main in. An older revision than the latest seen is left. For each
|
||||
// repository the source names, each module built from it whose fingerprint was computed at the head the source names is
|
||||
// written again with Src the revision; a module computed at another commit is left (a merge heard later is stamped
|
||||
// when it is written, applyMerged), and a repository none of the indexer's modules is built from writes nothing.
|
||||
func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Source) error {
|
||||
ix.mu.Lock()
|
||||
if rev < ix.source.rev {
|
||||
ix.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
ix.source = sourceNow{rev: rev, heads: maps.Clone(s.Heads)}
|
||||
ix.mu.Unlock()
|
||||
if ix.Modules == nil {
|
||||
return nil
|
||||
}
|
||||
repositories := slices.Sorted(maps.Keys(s.Heads))
|
||||
for _, repository := range repositories {
|
||||
head := s.Heads[repository]
|
||||
for _, k := range ix.Builds.Modules(repository) {
|
||||
key, err := delivery.ModulesKey(k.Module)
|
||||
if err != nil {
|
||||
ix.logf("%s: the module %q has no usable key, skipped: %v", repository, k.Module, err)
|
||||
continue
|
||||
}
|
||||
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur, err := delivery.DecodeModule(value)
|
||||
if err != nil {
|
||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||
return nil, false, nil
|
||||
}
|
||||
if cur.Repository != repository || cur.Commit != head || cur.Src == rev {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur.Src = rev
|
||||
data, err := delivery.EncodeModule(cur)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const otherMerge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
|
||||
func sourceOf(repo, head string) delivery.Source {
|
||||
return delivery.Source{Group: "g", Heads: map[string]string{repo: head}}
|
||||
}
|
||||
|
||||
func TestASourceRevisionStampsTheModulesOfTheRepositoriesHeadItNames(t *testing.T) { // 1344/AC1
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs),
|
||||
rec(t, "build-2", "c", "novox/mesh-catalog", "modules/c", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := moduleOf(t, mods, "a")
|
||||
if before.Src != 0 {
|
||||
t.Fatalf("module before the source = %+v, want Src 0", before)
|
||||
}
|
||||
// The source names the head merged for the delivery repository, and another repository's head no module of ours is built from.
|
||||
s := delivery.Source{Group: "g", Heads: map[string]string{"novox/mesh-delivery": merge, "novox/mesh-catalog": otherMerge}}
|
||||
if err := ix.ApplySource(ctx, 7, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
want := before
|
||||
want.Src = 7
|
||||
if !ok || !reflect.DeepEqual(m, want) {
|
||||
t.Errorf("module a = %+v (%v), want %+v: the fingerprint and the commit as the merge wrote them, with Src 7", m, ok, want)
|
||||
}
|
||||
if c, ok := moduleOf(t, mods, "c"); ok {
|
||||
t.Errorf("module c = %+v: written for a head no merge message of the indexer carried", c)
|
||||
}
|
||||
// The same revision again changes nothing; a later revision moves Src.
|
||||
if err := ix.ApplySource(ctx, 7, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.ApplySource(ctx, 9, s); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
|
||||
t.Errorf("module a Src = %d, want 9", m.Src)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeHeardAfterTheSourceNamesItsHeadIsStampedWhenWritten(t *testing.T) { // 1344/AC1 (the source first)
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := moduleOf(t, mods, "a"); ok {
|
||||
t.Fatal("a module was written from a source revision alone: its tree is the merge message's")
|
||||
}
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 7 || m.Commit != merge || m.FP == "" {
|
||||
t.Errorf("module a = %+v (%v), want the merge's fingerprint with Src 7", m, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeThatNoSourceRevisionNamesIsWrittenAsToday(t *testing.T) { // 1344/AC2
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
// No source revision yet.
|
||||
if err := ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "a"); !ok || m.Src != 0 || m.FP == "" {
|
||||
t.Fatalf("module a = %+v (%v), want a fingerprint with Src 0", m, ok)
|
||||
}
|
||||
// A source revision that names another head of the repository: a merge of the head it does not name keeps Src 0.
|
||||
if err := ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", otherMerge)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 0 {
|
||||
t.Errorf("module a Src = %d after a source naming another head, want 0", m.Src)
|
||||
}
|
||||
merged := mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newertree"})
|
||||
merged.Merge = "cccccccccccccccccccccccccccccccccccccccc"
|
||||
if err := ix.Apply(ctx, merged); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 0 || m.Commit != merged.Merge {
|
||||
t.Errorf("module a = %+v, want the new merge with Src 0", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestASourceRevisionOlderThanTheOneHeldIsLeft(t *testing.T) {
|
||||
ix, mods := modulesIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ctx := context.Background()
|
||||
_ = ix.Apply(ctx, mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"}))
|
||||
_ = ix.ApplySource(ctx, 9, sourceOf("novox/mesh-delivery", merge))
|
||||
_ = ix.ApplySource(ctx, 7, sourceOf("novox/mesh-delivery", merge))
|
||||
if m, _ := moduleOf(t, mods, "a"); m.Src != 9 {
|
||||
t.Errorf("module a Src = %d, want 9", m.Src)
|
||||
}
|
||||
}
|
||||
Vendored
+69
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"module": "audit-logger",
|
||||
"version": "1",
|
||||
"slug": "audit",
|
||||
"consumes": [
|
||||
"**"
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "code",
|
||||
"kind": "bundle",
|
||||
"language": "typescript",
|
||||
"entrypoints": [
|
||||
"index.js",
|
||||
"tools/index.js"
|
||||
],
|
||||
"loads": [
|
||||
"index.js",
|
||||
"tools/index.js"
|
||||
],
|
||||
"env": {
|
||||
"AUDIT_LOG": "${dir:trail}/audit.log",
|
||||
"AUDIT_SPOOL": "${dir:spool}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "trail",
|
||||
"path": "${dir:trail}",
|
||||
"class": "valuable",
|
||||
"why": "the audit trail, written nowhere else"
|
||||
},
|
||||
{
|
||||
"id": "spool",
|
||||
"path": "${dir:spool}",
|
||||
"class": "valuable",
|
||||
"why": "events the trail could not take yet; after the bus gives one up, the only copy"
|
||||
}
|
||||
]
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "trail",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "spool",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"unknown": "program"
|
||||
}
|
||||
],
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
]
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"module": "avahi",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"package-manager",
|
||||
"service-manager"
|
||||
],
|
||||
"tools": [
|
||||
"avahi_status",
|
||||
"avahi_browse",
|
||||
"avahi_resolve",
|
||||
"avahi_services"
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "avahi"
|
||||
},
|
||||
{
|
||||
"id": "daemon",
|
||||
"type": "service",
|
||||
"unit": "avahi-daemon.service",
|
||||
"state": "running",
|
||||
"boot": "enabled"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/avahi-tools",
|
||||
"binary": "avahi-tools",
|
||||
"loads": [
|
||||
"avahi-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Vendored
+66
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"module": "ca-trust",
|
||||
"version": "1",
|
||||
"slug": "catrust",
|
||||
"capabilities": [
|
||||
"service-manager"
|
||||
],
|
||||
"requires": [
|
||||
"internal-acme-ca"
|
||||
],
|
||||
"seats": [
|
||||
{
|
||||
"name": "the-mesh-trust-anchor",
|
||||
"scope": "node"
|
||||
}
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "the-mesh-trust-anchor",
|
||||
"scope": "node"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "anchor",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/anchor",
|
||||
"mode": "0755",
|
||||
"template": "files/anchor"
|
||||
},
|
||||
{
|
||||
"id": "unit",
|
||||
"type": "file",
|
||||
"path": "/etc/systemd/system/mesh-ca-trust.service",
|
||||
"mode": "0644",
|
||||
"template": "files/mesh-ca-trust.service"
|
||||
},
|
||||
{
|
||||
"id": "trust",
|
||||
"type": "service",
|
||||
"unit": "mesh-ca-trust.service",
|
||||
"state": "running",
|
||||
"boot": "enabled",
|
||||
"restart-on": [
|
||||
"anchor",
|
||||
"unit"
|
||||
]
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Vendored
+57
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"module": "netcheck",
|
||||
"version": "1",
|
||||
"emits": [
|
||||
"tunnel.endpoint-changed",
|
||||
"tunnel.handshake-stale",
|
||||
"tunnel.handshake-fresh"
|
||||
],
|
||||
"tools": [
|
||||
"netcheck_tunnel",
|
||||
"netcheck_tcp",
|
||||
"netcheck_dns",
|
||||
"netcheck_http",
|
||||
"netcheck_listening"
|
||||
],
|
||||
"replaces": {
|
||||
"netcheck_listening": [
|
||||
"ss -lunt",
|
||||
"ss -ltn",
|
||||
"netstat -lnt"
|
||||
],
|
||||
"netcheck_dns": [
|
||||
"dig",
|
||||
"nslookup"
|
||||
],
|
||||
"netcheck_tunnel": [
|
||||
"wg show",
|
||||
"wg show mesh0 dump"
|
||||
]
|
||||
},
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools-go",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/netcheck",
|
||||
"binary": "netcheck",
|
||||
"loads": [
|
||||
"netcheck"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "tools-typescript",
|
||||
"kind": "bundle",
|
||||
"language": "typescript",
|
||||
"entrypoints": [
|
||||
"tools/index.js"
|
||||
],
|
||||
"loads": [
|
||||
"tools/index.js"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Vendored
+140
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"module": "build-agent",
|
||||
"version": "1",
|
||||
"slug": "agent",
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-build-agent",
|
||||
"scope": "node",
|
||||
"serves": ["current", "kill", "pause", "resume"]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.pulls",
|
||||
"mesh-delivery.pulls"
|
||||
],
|
||||
"requires": [
|
||||
"artifact-store",
|
||||
{
|
||||
"name": "npm-package-registry",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"binds": {
|
||||
"npm-package-registry": "${dir:mesh-state}/package-registry.json"
|
||||
},
|
||||
"secrets": {
|
||||
"npm-package-registry": "${dir:mesh-state}/package-registry.secret"
|
||||
},
|
||||
"own-secrets": {
|
||||
"broker": "${dir:mesh-state}/broker",
|
||||
"genesis-package-registry": {
|
||||
"path": "${dir:mesh-state}/genesis-package-registry.json",
|
||||
"issued-by": "outside"
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "workspace",
|
||||
"path": "${dir:workspace}",
|
||||
"class": "cache",
|
||||
"why": "a build's working copy, cloned again for every build"
|
||||
},
|
||||
{
|
||||
"id": "judge-cache",
|
||||
"path": "${dir:judge-cache}",
|
||||
"class": "cache",
|
||||
"why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)"
|
||||
}
|
||||
]
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "mesh-state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": "mesh"
|
||||
},
|
||||
{
|
||||
"id": "workspace",
|
||||
"type": "directory",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "judge-cache",
|
||||
"type": "directory",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "agent-env",
|
||||
"type": "file",
|
||||
"path": "${dir:mesh-state}/build-agent.env",
|
||||
"mode": "0600",
|
||||
"template": "files/build-agent.env"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
"type": "container",
|
||||
"name": "mesh-build-agent",
|
||||
"artifact": "server",
|
||||
"env-file": [
|
||||
"${dir:mesh-state}/build-agent.env"
|
||||
],
|
||||
"volumes": [
|
||||
"${dir:mesh-state}:/run/mesh:ro",
|
||||
"${dir:workspace}:${dir:workspace}",
|
||||
"${dir:judge-cache}:${dir:judge-cache}",
|
||||
"/var/run/docker.sock:/var/run/docker.sock"
|
||||
],
|
||||
"restart-on": [
|
||||
"agent-env"
|
||||
],
|
||||
"network": "host",
|
||||
"health": {
|
||||
"kind": "exec",
|
||||
"command": "/usr/local/bin/mesh-builder ready",
|
||||
"interval": "10s",
|
||||
"timeout": "5s",
|
||||
"looks": 3,
|
||||
"grace": "60s"
|
||||
},
|
||||
"busy": {
|
||||
"command": "/usr/local/bin/mesh-builder busy"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "server",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile",
|
||||
"compiles": "cmd/mesh-builder",
|
||||
"context": {
|
||||
"seat": "git",
|
||||
"repository": "novox/mesh-controller",
|
||||
"ref": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
],
|
||||
"on": [
|
||||
{
|
||||
"arg": "GO_BASE",
|
||||
"image": "golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c"
|
||||
},
|
||||
{
|
||||
"arg": "ALPINE_BASE",
|
||||
"image": "alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The new-form fingerprint of a module is written, and written again with no merge, when what it is of changes
|
||||
// (novox/hq ADR 0465 clause 2; issue 1444): a module's tree and lock hashes come with a merge and are kept in
|
||||
// modules.<m>.Inputs2; its builder's, its bases' and its toolchain's come from here. The indexer holds no graph, so each
|
||||
// recompute reads the manifest of every module from manifests.<digest> and takes the modules in the order their bases
|
||||
// and the builder come first.
|
||||
|
||||
const (
|
||||
// BuilderModule is the module that builds every module (novox/hq ADR 0453 clause 5): each module's Builder input is
|
||||
// its new-form fingerprint. It does not exist before release R4c-4, and a module then has none for want of it.
|
||||
BuilderModule = "mesh-delivery-builder"
|
||||
// BuilderBuiltBy is the module that builds BuilderModule until release R4g: its fingerprint of today's form is the
|
||||
// builder's own Builder input, so a change of build-agent moves the builder's value (ADR 0453 clause 5).
|
||||
BuilderBuiltBy = "build-agent"
|
||||
)
|
||||
|
||||
// ToolchainModules are the build module of each language, whose state toolchains (delivery.ToolchainsBucket) the
|
||||
// indexer reads: the language's agreed toolchain is the one every node's entry in it agrees on.
|
||||
var ToolchainModules = map[string]string{"go": "mesh-build-go", "typescript": "mesh-build-node"}
|
||||
|
||||
// ToolchainStreams are the stream each build module carries its toolchain.reported event on, by module, as the bus names it
|
||||
// (a stream's local name in capitals, catalogue.StreamBusName): `toolchains-go` and `toolchains-node`, declared by the
|
||||
// build modules (streams, with the event toolchain.reported) and heard by this module (consumes).
|
||||
var ToolchainStreams = map[string]string{"mesh-build-go": "TOOLCHAINS_GO", "mesh-build-node": "TOOLCHAINS_NODE"}
|
||||
|
||||
// PutToolchain takes one change of a build module's toolchains bucket in, without writing any module: a put of a node's
|
||||
// toolchain of a language (key <node>.<language>, delivery.ToolchainKey), or its delete. It answers whether what the
|
||||
// language's toolchain is of (version and recipe) changed. A key, or a value, that is not a toolchain of the module's
|
||||
// language is said in the log and left out.
|
||||
func (ix *Indexer) PutToolchain(module, key string, value []byte, deleted bool) bool {
|
||||
language := ""
|
||||
for l, m := range ToolchainModules {
|
||||
if m == module {
|
||||
language = l
|
||||
}
|
||||
}
|
||||
if language == "" {
|
||||
ix.logf("toolchains of %q are not read: it is no build module the indexer knows", module)
|
||||
return false
|
||||
}
|
||||
node, keyLanguage, err := delivery.ParseToolchainKey(key)
|
||||
if err != nil || keyLanguage != language {
|
||||
ix.logf("%s_toolchains.%s is not a %s toolchain's key, left out: %v", module, key, language, err)
|
||||
return false
|
||||
}
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
if deleted {
|
||||
if _, had := ix.toolchains[language][node]; !had {
|
||||
return false
|
||||
}
|
||||
delete(ix.toolchains[language], node)
|
||||
return true
|
||||
}
|
||||
tc, err := delivery.DecodeToolchain(value)
|
||||
if err != nil || tc.Language != language {
|
||||
ix.logf("%s_toolchains.%s is not a %s toolchain, left out: %v", module, key, language, err)
|
||||
return false
|
||||
}
|
||||
if ix.toolchains == nil {
|
||||
ix.toolchains = map[string]map[string]delivery.Toolchain{}
|
||||
}
|
||||
if ix.toolchains[language] == nil {
|
||||
ix.toolchains[language] = map[string]delivery.Toolchain{}
|
||||
}
|
||||
cur, had := ix.toolchains[language][node]
|
||||
ix.toolchains[language][node] = tc
|
||||
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
|
||||
}
|
||||
|
||||
// ApplyToolchain takes one change of a build module's toolchains in (PutToolchain) and, when the language's toolchain
|
||||
// changed, writes the modules again that it moves (Recompute). The same version and recipe seen again writes nothing.
|
||||
func (ix *Indexer) ApplyToolchain(ctx context.Context, module, key string, value []byte, deleted bool) error {
|
||||
if !ix.PutToolchain(module, key, value, deleted) {
|
||||
return nil
|
||||
}
|
||||
return ix.Recompute(ctx)
|
||||
}
|
||||
|
||||
// agreed is the toolchain of each language every reporting node agrees on, or why there is none.
|
||||
func (ix *Indexer) agreed() map[string]toolchainOrNone {
|
||||
ix.mu.Lock()
|
||||
defer ix.mu.Unlock()
|
||||
out := map[string]toolchainOrNone{}
|
||||
for language := range ToolchainModules {
|
||||
var entries []delivery.NodeToolchain
|
||||
for _, node := range slices.Sorted(maps.Keys(ix.toolchains[language])) {
|
||||
entries = append(entries, delivery.NodeToolchain{Node: node, Toolchain: ix.toolchains[language][node]})
|
||||
}
|
||||
tc, none := delivery.AgreedToolchain(entries)
|
||||
out[language] = toolchainOrNone{tc, none}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type toolchainOrNone struct {
|
||||
tc delivery.Toolchain2
|
||||
none string
|
||||
}
|
||||
|
||||
// known is a module the indexer holds the inputs of: its record as read, the manifest's shape and the key it is written at.
|
||||
type known struct {
|
||||
key string
|
||||
cur delivery.Module
|
||||
languages map[string]string
|
||||
bases []string
|
||||
contexts bool
|
||||
}
|
||||
|
||||
// Recompute writes again every module whose new-form fingerprint, or reason for none, is no longer what it holds: from
|
||||
// the inputs kept at its merge (Inputs2), its manifest (manifests.<digest>), the fingerprints of its bases and of the
|
||||
// builder, and the toolchains the build modules report. A module is computed after its bases and the builder, whatever
|
||||
// the order the bucket lists them in; modules that stand on each other have a cycle as their reason. A module is written
|
||||
// only when its fingerprint or its reason changes, and only if its manifest is still the one computed from. A module with
|
||||
// no kept inputs (a message that carried no manifest) is left as it is.
|
||||
func (ix *Indexer) Recompute(ctx context.Context) error { return ix.recomputeWith(ctx, nil) }
|
||||
|
||||
// recomputeWith is Recompute for a merge as well: the records the merge gives (overlay, by module key) stand in place
|
||||
// of what the bucket holds, are computed with the others, and are written once, with their final fingerprint or reason
|
||||
// (issue 1444: a merge writes each module once). A record of the overlay that has no kept inputs is written as it is.
|
||||
func (ix *Indexer) recomputeWith(ctx context.Context, overlay map[string]delivery.Module) error {
|
||||
if ix.Modules == nil {
|
||||
return nil
|
||||
}
|
||||
if ix.Manifests == nil {
|
||||
return ix.writeOverlay(ctx, overlay, nil)
|
||||
}
|
||||
ix.recompute.Lock()
|
||||
defer ix.recompute.Unlock()
|
||||
keys, err := ix.Modules.Keys(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing the modules: %w", err)
|
||||
}
|
||||
for key := range overlay {
|
||||
if !slices.Contains(keys, key) {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
slices.Sort(keys)
|
||||
held := map[string]delivery.Module{}
|
||||
all := map[string]*known{}
|
||||
for _, key := range keys {
|
||||
cur, ok := overlay[key]
|
||||
if !ok {
|
||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
if cur, err = delivery.DecodeModule(value); err != nil {
|
||||
ix.logf("modules.%s holds what is not a module, left: %v", key, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
held[key] = cur
|
||||
if cur.Manifest == "" || cur.Inputs2 == nil {
|
||||
continue
|
||||
}
|
||||
m, err := ix.manifestOf(ctx, cur.Manifest)
|
||||
if err != nil {
|
||||
ix.logf("modules.%s: its manifest %s cannot be read, left: %v", key, cur.Manifest, err)
|
||||
continue
|
||||
}
|
||||
all[key] = &known{key: key, cur: cur, languages: languagesOf(m), bases: basesOf(m), contexts: hasContext(m)}
|
||||
}
|
||||
tools := ix.agreed()
|
||||
type result struct {
|
||||
fp, none string
|
||||
tcs map[string]delivery.ToolchainUsed // by language, when there is a fingerprint
|
||||
}
|
||||
done := map[string]result{}
|
||||
visiting := map[string]bool{}
|
||||
var of func(module string, chain []string) result
|
||||
of = func(module string, chain []string) result {
|
||||
if r, ok := done[module]; ok {
|
||||
return r
|
||||
}
|
||||
k, ours := all[module]
|
||||
if !ours { // a module the indexer holds no inputs of: its record, as written
|
||||
cur, has := held[module]
|
||||
switch {
|
||||
case !has:
|
||||
return result{fp: "", none: "the indexer holds no record of it"}
|
||||
case cur.FP2 != "":
|
||||
return result{fp: cur.FP2, none: ""}
|
||||
case cur.None2 != "":
|
||||
return result{fp: "", none: cur.None2}
|
||||
}
|
||||
return result{fp: "", none: "its record carries no fingerprint of the new form"}
|
||||
}
|
||||
if visiting[module] {
|
||||
return result{fp: "", none: "a cycle of bases: " + strings.Join(append(chain, module), " -> ")}
|
||||
}
|
||||
visiting[module] = true
|
||||
defer delete(visiting, module)
|
||||
chain = append(chain, module)
|
||||
kept := *k.cur.Inputs2
|
||||
kept.Bases, kept.Builder = nil, ""
|
||||
var missing []string
|
||||
for _, b := range k.bases {
|
||||
if r := of(b, chain); r.fp != "" {
|
||||
kept.Bases = append(kept.Bases, r.fp)
|
||||
} else {
|
||||
missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", b, r.none))
|
||||
}
|
||||
}
|
||||
slices.Sort(kept.Bases)
|
||||
slices.Sort(missing)
|
||||
toolchains := map[string]delivery.Toolchain2{}
|
||||
var noToolchain string
|
||||
for _, artifact := range slices.Sorted(maps.Keys(k.languages)) {
|
||||
t := tools[k.languages[artifact]]
|
||||
if t.none != "" {
|
||||
if noToolchain == "" {
|
||||
noToolchain = t.none
|
||||
}
|
||||
continue
|
||||
}
|
||||
toolchains[artifact] = t.tc
|
||||
}
|
||||
var noBuilder string
|
||||
if module == BuilderModule {
|
||||
// Built by build-agent until release R4g: its value of today's form.
|
||||
if a := held[BuilderBuiltBy]; a.FP != "" {
|
||||
kept.Builder = a.FP
|
||||
} else {
|
||||
noBuilder = fmt.Sprintf("%s, which builds it, has no fingerprint (%s)", BuilderBuiltBy, a.None)
|
||||
}
|
||||
} else if r := of(BuilderModule, chain); r.fp != "" {
|
||||
kept.Builder = r.fp
|
||||
} else {
|
||||
noBuilder = None2Builder
|
||||
if _, has := held[BuilderModule]; has {
|
||||
noBuilder += ": " + r.none
|
||||
}
|
||||
}
|
||||
var r result
|
||||
switch {
|
||||
case k.contexts:
|
||||
r.none = None2Context
|
||||
case len(missing) > 0:
|
||||
r.none = strings.Join(missing, "; ")
|
||||
case noToolchain != "":
|
||||
r.none = noToolchain
|
||||
case noBuilder != "":
|
||||
r.none = noBuilder
|
||||
default:
|
||||
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
|
||||
if r.fp != "" {
|
||||
r.tcs = toolchainsUsed(toolchains)
|
||||
}
|
||||
}
|
||||
done[module] = r
|
||||
k.cur.Inputs2 = &kept
|
||||
return r
|
||||
}
|
||||
finals := map[string]delivery.Module{}
|
||||
for _, key := range slices.Sorted(maps.Keys(all)) {
|
||||
k := all[key]
|
||||
r := of(key, nil)
|
||||
if _, merged := overlay[key]; merged {
|
||||
final := overlay[key]
|
||||
final.FP2, final.None2, final.Inputs2, final.Toolchains = r.fp, r.none, k.cur.Inputs2, r.tcs
|
||||
// What the bases and the builder gave is kept as it was while the value it gave does not change, so a merge
|
||||
// that moves nothing for the module writes nothing (as Recompute does).
|
||||
if value, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
|
||||
if old, err := delivery.DecodeModule(value); err == nil && old.Inputs2 != nil && old.FP2 == r.fp && old.None2 == r.none {
|
||||
in := *final.Inputs2
|
||||
in.Bases, in.Builder = old.Inputs2.Bases, old.Inputs2.Builder
|
||||
final.Inputs2 = &in
|
||||
}
|
||||
}
|
||||
finals[key] = final
|
||||
continue
|
||||
}
|
||||
if k.cur.FP2 == r.fp && k.cur.None2 == r.none && reflect.DeepEqual(k.cur.Toolchains, r.tcs) {
|
||||
continue
|
||||
}
|
||||
digest, inputs := k.cur.Manifest, k.cur.Inputs2
|
||||
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if !found {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur, err := delivery.DecodeModule(value)
|
||||
if err != nil || cur.Manifest != digest || cur.Inputs2 == nil {
|
||||
return nil, false, nil // a merge changed it meanwhile; its own recompute follows
|
||||
}
|
||||
if cur.FP2 == r.fp && cur.None2 == r.none && reflect.DeepEqual(cur.Toolchains, r.tcs) {
|
||||
return nil, false, nil
|
||||
}
|
||||
cur.FP2, cur.None2, cur.Toolchains = r.fp, r.none, r.tcs
|
||||
cur.Inputs2 = inputs
|
||||
data, err := delivery.EncodeModule(cur)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
return ix.writeOverlay(ctx, overlay, finals)
|
||||
}
|
||||
|
||||
// writeOverlay writes the records of a merge, each once and only if it differs from what the bucket holds; finals has
|
||||
// the ones that were computed, the others are written as the merge gave them.
|
||||
func (ix *Indexer) writeOverlay(ctx context.Context, overlay, finals map[string]delivery.Module) error {
|
||||
for _, key := range slices.Sorted(maps.Keys(overlay)) {
|
||||
next := overlay[key]
|
||||
if f, ok := finals[key]; ok {
|
||||
next = f
|
||||
}
|
||||
err := write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
data, err := delivery.EncodeModule(next)
|
||||
return data, false, err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("modules.%s: %w", key, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// manifestOf is the parsed manifest manifests.<digest> holds.
|
||||
func (ix *Indexer) manifestOf(ctx context.Context, digest string) (catalogue.Manifest, error) {
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
value, _, found, err := ix.Manifests.Get(ctx, key)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
if !found {
|
||||
return catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", key)
|
||||
}
|
||||
kept, err := delivery.DecodeManifest(value)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, err
|
||||
}
|
||||
return catalogue.ParseManifest([]byte(kept.JSON))
|
||||
}
|
||||
|
||||
// hasContext is whether an artifact of the manifest reads another repository's tree.
|
||||
func hasContext(m catalogue.Manifest) bool {
|
||||
if m.Build == nil {
|
||||
return false
|
||||
}
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Context != nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// toolchainsUsed are the toolchains a fingerprint was computed with, by language (an artifact's toolchain is its language's).
|
||||
func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.ToolchainUsed {
|
||||
if len(by) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := map[string]delivery.ToolchainUsed{}
|
||||
for _, t := range by {
|
||||
out[t.Language] = delivery.ToolchainUsed{Version: t.Version, Recipe: t.Recipe}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The fixtures here are mesh-catalog a67f0196's manifests (testdata, captured from that commit) with the module renamed (and,
|
||||
// where a test needs one, a build.on entry added as the controller's own test has it, standing_on_test.go:22): the manifest
|
||||
// parser, not a hand-written manifest, reads them.
|
||||
|
||||
func renamed(t *testing.T, from, to string) string {
|
||||
t.Helper()
|
||||
raw := catalogueManifest(t, from)
|
||||
out := strings.Replace(raw, `"module": "`+from+`"`, `"module": "`+to+`"`, 1)
|
||||
if out == raw {
|
||||
t.Fatalf("%s's manifest has no module line", from)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func standingOn(t *testing.T, raw, base string) string {
|
||||
t.Helper()
|
||||
out := strings.Replace(raw, `"build": {`, `"build": {
|
||||
"on": [{"arg": "RUNTIME_BASE", "module": "`+base+`", "artifact": "runtime"}],`, 1)
|
||||
if out == raw {
|
||||
t.Fatal("the manifest has no build")
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
const buildAgentRepo = "novox/mesh-controller"
|
||||
|
||||
type mod struct {
|
||||
name, manifest string
|
||||
}
|
||||
|
||||
// merged applies one merge of the catalogue carrying these modules (every one a Go bundle's go.sum, a TypeScript bundle's
|
||||
// package-lock.json) and returns the indexer.
|
||||
func newFormIndexer(t *testing.T, mods ...mod) (*Indexer, *memStore) {
|
||||
t.Helper()
|
||||
ix, store, f := newFormIndexerUnmerged(t, mods...)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// build-agent's value of today's form, from a merge of the controller (modules.build-agent holds its fingerprint).
|
||||
if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-1"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return ix, store
|
||||
}
|
||||
|
||||
// newFormIndexerUnmerged is the indexer and the catalogue merge of these modules, not yet applied.
|
||||
func newFormIndexerUnmerged(t *testing.T, mods ...mod) (*Indexer, *memStore, delivery.Forge) {
|
||||
t.Helper()
|
||||
var recs [][2]any
|
||||
trees, manifests := map[string]string{}, map[string]string{}
|
||||
locks := map[string]map[string]string{}
|
||||
for i, m := range mods {
|
||||
dir := "modules/" + m.name
|
||||
recs = append(recs, rec(t, "build-"+m.name, m.name, catalogueRepo, dir, "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
trees[dir], manifests[dir] = "tree-"+m.name, m.manifest
|
||||
locks[dir] = map[string]string{"go.sum": avahiGoSum, "package-lock.json": strings.Repeat("c", 63) + string(rune('a'+i))}
|
||||
}
|
||||
recs = append(recs, rec(t, "build-agent-1", "build-agent", buildAgentRepo, "modules/build-agent", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ix, store, _ := manifestsIndexer(t, recs...)
|
||||
return ix, store, withManifests(mergedMsg(catalogueRepo, trees), manifests, locks)
|
||||
}
|
||||
|
||||
// toolchainValue is a toolchains value as the build module's writer encodes it (mesh-sdk go/delivery EncodeToolchain).
|
||||
func toolchainValue(t *testing.T, language, version, recipe string) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now()})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func reportToolchain(t *testing.T, ix *Indexer, module, node, language, version string) {
|
||||
t.Helper()
|
||||
key, err := delivery.ToolchainKey(node, language)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ix.ApplyToolchain(context.Background(), module, key, toolchainValue(t, language, version, "1"), false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 1444/AC5
|
||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.None2 != delivery.ToolchainNoReport {
|
||||
t.Fatalf("before: none2 = %q", m.None2)
|
||||
}
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1")
|
||||
m, _ := moduleOf(t, mods, "avahi")
|
||||
if m.None2 != None2Builder || m.FP2 != "" {
|
||||
t.Errorf("none2 = %q fp2 = %q, want none2 %q", m.None2, m.FP2, None2Builder)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesReportingDifferentVersionsLeaveTheReasonNamingBoth(t *testing.T) { // 1444/AC6
|
||||
ix, mods := newFormIndexer(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
|
||||
m, _ := moduleOf(t, mods, "avahi")
|
||||
if !strings.Contains(m.None2, "builders disagree") || !strings.Contains(m.None2, "go1.27.1") || !strings.Contains(m.None2, "go1.26.0") {
|
||||
t.Errorf("none2 = %q, want 'builders disagree' naming both versions", m.None2)
|
||||
}
|
||||
}
|
||||
|
||||
// builderModules are the builder, a base, a module on the base (named so that it is stored before its base) and a module
|
||||
// whose base is a TypeScript module with no toolchain reported.
|
||||
func builderModules(t *testing.T) []mod {
|
||||
return []mod{
|
||||
{"mesh-delivery-builder", renamed(t, "avahi", "mesh-delivery-builder")},
|
||||
{"mesh-tools", renamed(t, "avahi", "mesh-tools")},
|
||||
{"avahi", standingOn(t, catalogueManifest(t, "avahi"), "mesh-tools")},
|
||||
{"other", standingOn(t, renamed(t, "avahi", "other"), "ts-base")},
|
||||
{"ts-base", renamed(t, "audit-logger", "ts-base")},
|
||||
}
|
||||
}
|
||||
|
||||
func goInputs(t *testing.T, ix *Indexer, mods *memStore, module, builder string, bases ...string) delivery.SourceInputs2 {
|
||||
t.Helper()
|
||||
return delivery.SourceInputs2{Module: module, Tree: "tree-" + module, Builder: builder, Bases: bases,
|
||||
Locks: map[string]string{"tools": avahiGoSum},
|
||||
Toolchains: map[string]delivery.Toolchain2{"tools": {Language: "go", Version: "go1.27.1", Recipe: "1"}}}
|
||||
}
|
||||
|
||||
func fp2(t *testing.T, in delivery.SourceInputs2) string {
|
||||
t.Helper()
|
||||
fp, none := delivery.SourceFingerprint2(in)
|
||||
if none != "" {
|
||||
t.Fatal(none)
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
func TestBasesAndTheBuilderAreComputedBeforeTheModulesTakingThem(t *testing.T) { // 1444/AC7, AC8
|
||||
ix, mods := newFormIndexer(t, builderModules(t)...)
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
agent, _ := moduleOf(t, mods, "build-agent")
|
||||
if agent.FP == "" {
|
||||
t.Fatalf("build-agent has no fingerprint of today's form: %+v", agent)
|
||||
}
|
||||
builderFP := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agent.FP))
|
||||
toolsFP := fp2(t, goInputs(t, ix, mods, "mesh-tools", builderFP))
|
||||
want := map[string]string{"mesh-delivery-builder": builderFP, "mesh-tools": toolsFP, "avahi": fp2(t, goInputs(t, ix, mods, "avahi", builderFP, toolsFP))}
|
||||
for name, fp := range want {
|
||||
if m, _ := moduleOf(t, mods, name); m.FP2 != fp || m.None2 != "" {
|
||||
t.Errorf("%s: fp2 = %q none2 = %q, want %s", name, m.FP2, m.None2, fp)
|
||||
}
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") {
|
||||
t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2)
|
||||
}
|
||||
// 1444/AC8: build-agent changes at a merge of the controller; the builder's value moves with it, and so do the modules built by it.
|
||||
if err := ix.Apply(context.Background(), mergedMsg(buildAgentRepo, map[string]string{"modules/build-agent": "agent-tree-2"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved, _ := moduleOf(t, mods, "build-agent")
|
||||
if moved.FP == agent.FP {
|
||||
t.Fatal("build-agent's fingerprint did not change")
|
||||
}
|
||||
newBuilder := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", moved.FP))
|
||||
if m, _ := moduleOf(t, mods, "mesh-delivery-builder"); m.FP2 != newBuilder || newBuilder == builderFP {
|
||||
t.Errorf("builder: fp2 = %q, want %s (was %s)", m.FP2, newBuilder, builderFP)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != fp2(t, goInputs(t, ix, mods, "mesh-tools", newBuilder)) {
|
||||
t.Errorf("mesh-tools did not follow the builder: %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAToolchainEntryChangedToTheSameValuesWritesNothing(t *testing.T) { // 1444/AC9
|
||||
ix, mods := newFormIndexer(t, builderModules(t)...)
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
before := mods.writes
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") // the same version and recipe, a later time
|
||||
if mods.writes != before {
|
||||
t.Errorf("modules written %d times after the same values, want none", mods.writes-before)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModulesThatStandOnEachOtherAreNamedAsACycle(t *testing.T) { // 1444 (the indexer has no graph: a cycle is a reason)
|
||||
ix, mods := newFormIndexer(t,
|
||||
mod{"aa", standingOn(t, renamed(t, "avahi", "aa"), "bb")},
|
||||
mod{"bb", standingOn(t, renamed(t, "avahi", "bb"), "aa")})
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
for _, name := range []string{"aa", "bb"} {
|
||||
if m, _ := moduleOf(t, mods, name); m.FP2 != "" || !strings.Contains(m.None2, "cycle") {
|
||||
t.Errorf("%s: fp2 = %q none2 = %q, want a reason naming the cycle", name, m.FP2, m.None2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeWritesEachModuleOnceWithItsFinalValueAndAReplayWritesNothing(t *testing.T) { // 1444 (review finding 8)
|
||||
ctx := context.Background()
|
||||
ix, mods, f := newFormIndexerUnmerged(t, mod{"avahi", catalogueManifest(t, "avahi")})
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") // agreed before the merge is heard
|
||||
if err := ix.Apply(ctx, f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := moduleOf(t, mods, "avahi")
|
||||
if mods.writes != 1 || m.None2 != None2Builder {
|
||||
t.Errorf("writes = %d none2 = %q after one merge, want one write with %q", mods.writes, m.None2, None2Builder)
|
||||
}
|
||||
if err := ix.Apply(ctx, f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if mods.writes != 1 {
|
||||
t.Errorf("writes = %d after the same merge again, want still 1", mods.writes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergeKeepsAFingerprintTheModuleStillHas(t *testing.T) { // 1444 (review finding 8: a merge never blanks FP2)
|
||||
ctx := context.Background()
|
||||
ix, mods := newFormIndexer(t, builderModules(t)...)
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
before, _ := moduleOf(t, mods, "mesh-tools")
|
||||
if before.FP2 == "" {
|
||||
t.Fatalf("mesh-tools has no fp2: %+v", before)
|
||||
}
|
||||
w := mods.writes
|
||||
// the same catalogue merged again at a later commit: nothing moved
|
||||
_, _, f := newFormIndexerUnmerged(t, builderModules(t)...)
|
||||
if err := ix.Apply(ctx, f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if after, _ := moduleOf(t, mods, "mesh-tools"); after.FP2 != before.FP2 || mods.writes != w {
|
||||
t.Errorf("fp2 = %q (was %q), writes %d more", after.FP2, before.FP2, mods.writes-w)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) { // 1615/AC3
|
||||
ix, mods := newFormIndexer(t, builderModules(t)...)
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.Toolchains != nil {
|
||||
t.Fatalf("before a toolchain is reported: toolchains = %+v, want none", m.Toolchains)
|
||||
}
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
|
||||
m, _ := moduleOf(t, mods, "avahi")
|
||||
want := map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}}
|
||||
if m.FP2 == "" || !reflect.DeepEqual(m.Toolchains, want) {
|
||||
t.Fatalf("avahi: fp2 %q toolchains %+v, want %+v", m.FP2, m.Toolchains, want)
|
||||
}
|
||||
if o, _ := moduleOf(t, mods, "other"); o.FP2 != "" || o.Toolchains != nil {
|
||||
t.Errorf("a module with no fingerprint carries toolchains %+v", o.Toolchains)
|
||||
}
|
||||
// An upgrade moves the fingerprint and the toolchains together, with no merge.
|
||||
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.2")
|
||||
n, _ := moduleOf(t, mods, "avahi")
|
||||
if n.FP2 == m.FP2 || n.Toolchains["go"].Version != "go1.27.2" {
|
||||
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.FP2, m.FP2, n.Toolchains)
|
||||
}
|
||||
// Nodes that disagree leave no fingerprint and no toolchains.
|
||||
reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.26.0")
|
||||
if d, _ := moduleOf(t, mods, "avahi"); d.FP2 != "" || d.Toolchains != nil {
|
||||
t.Errorf("disagreeing nodes: fp2 %q toolchains %+v, want neither", d.FP2, d.Toolchains)
|
||||
}
|
||||
}
|
||||
@@ -71,20 +71,20 @@ func sorted(s []string) []string {
|
||||
return s
|
||||
}
|
||||
|
||||
func TestManifestDeclarations(t *testing.T) { // 1238/AC1
|
||||
func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build modules' toolchains)
|
||||
m, top := load(t)
|
||||
if m.Module != "mesh-delivery-indexer" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-controller.builds", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
|
||||
// internal/broker/state.go:62-93).
|
||||
if want := []string{"mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
|
||||
@@ -3,13 +3,18 @@
|
||||
"version": "1",
|
||||
"slug": "indexer",
|
||||
"consumes": [
|
||||
"mesh-build-go.toolchain.reported",
|
||||
"mesh-build-node.toolchain.reported",
|
||||
"mesh-delivery-forge-bridge.forge.**"
|
||||
],
|
||||
"state": [
|
||||
{ "name": "candidates", "history": 10 },
|
||||
{ "name": "manifests", "history": 1 },
|
||||
{ "name": "modules", "history": 20 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-build-go.toolchains",
|
||||
"mesh-build-node.toolchains",
|
||||
"mesh-controller.builds",
|
||||
"mesh-delivery-gate.source"
|
||||
],
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
)
|
||||
|
||||
// The teller's three bus adapters for mesh-sdk go/asker, which in a bundle reaches the bus through the runtime: here the
|
||||
// teller is a process on its own credential. What each may do is the controller's composed grant for a module that uses
|
||||
// the seat operator-channel (mesh-control internal/broker seattraffic.go SeatTrafficOf): publish its own ask, cancel and
|
||||
// condition, hear its own decided event, and read the router's record of its own asks.
|
||||
|
||||
// callTimeout bounds a read of the router's record.
|
||||
const callTimeout = 30 * time.Second
|
||||
|
||||
// publishVia publishes an accept of the seat, de-duplicated by id: the same id twice is one message.
|
||||
func publishVia(ctx context.Context, js jetstream.JetStream) func(subject string, body any, id string) error {
|
||||
return func(subject string, body any, id string) error {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var opts []jetstream.PublishOpt
|
||||
if id != "" {
|
||||
opts = append(opts, jetstream.WithMsgID(id))
|
||||
}
|
||||
_, err = js.Publish(ctx, subject, data, opts...)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// recordVia reads the router's record of one of this module's asks: the key `<module>.<id>` of the router's bucket asks,
|
||||
// by a direct get on the subject the controller grants for it, `$JS.API.DIRECT.GET.KV_asks.$KV.asks.<module>.>`. The
|
||||
// record is {state, warrant}. An ask the router holds no record of is nil.
|
||||
func recordVia(ctx context.Context, conn *nats.Conn, module string) func(id string) (*stdio.StateEntry, error) {
|
||||
return func(id string) (*stdio.StateEntry, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, callTimeout)
|
||||
defer cancel()
|
||||
msg, err := conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_asks.$KV.asks."+module+"."+id, nil)
|
||||
if errors.Is(err, nats.ErrNoResponders) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status := msg.Header.Get("Status"); status == "404" || len(msg.Data) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return &stdio.StateEntry{Key: id, Value: msg.Data}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// decidedVia hears the warrants the router says to this module alone: its own `decided` event, a subject only the router
|
||||
// publishes (the grant to subscribe it names the module). The handler is the asker's: pattern is the key it expects.
|
||||
func decidedVia(conn *nats.Conn, module string, logf func(string, ...any)) func(pattern string, handle func(stdio.Envelope) error) error {
|
||||
return func(pattern string, handle func(stdio.Envelope) error) error {
|
||||
_, err := conn.Subscribe(asks.DecidedSubject(module), func(m *nats.Msg) {
|
||||
if err := handle(stdio.Envelope{Key: pattern, Body: m.Data}); err != nil {
|
||||
logf("a warrant could not be handled and is read again from the router's record at the next start: %v", err)
|
||||
}
|
||||
})
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// sayVia publishes a condition on the seat's one subject for conditions, under the module's own name.
|
||||
func sayVia(ctx context.Context, js jetstream.JetStream, module string) func(c asks.Condition) error {
|
||||
return func(c asks.Condition) error {
|
||||
subject, err := asks.ConditionSubject(module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := asks.EncodeCondition(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = js.Publish(ctx, subject, data)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// watchAsks hands each key of the gate's bucket asks, with its revision, to put, and each key taken away to gone, until
|
||||
// ctx ends or the watch fails.
|
||||
func watchAsks(ctx context.Context, kv jetstream.KeyValue, put func(key string, rev uint64, value []byte), gone func(key string)) error {
|
||||
w, err := kv.Watch(ctx, ">")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer w.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case e, ok := <-w.Updates():
|
||||
if !ok {
|
||||
return errors.New("the watch ended")
|
||||
}
|
||||
switch {
|
||||
case e == nil:
|
||||
case e.Operation() == jetstream.KeyValuePut:
|
||||
put(e.Key(), e.Revision(), e.Value())
|
||||
default:
|
||||
gone(e.Key())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,9 @@ import (
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
|
||||
@@ -121,3 +124,275 @@ func TestTellerWaitsForTheGate(t *testing.T) { // 1240/AC2
|
||||
t.Errorf("the teller stopped with %v, want a clean stop", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The principals of the composed accounts file for the teller's tests, with the permissions mesh-controller composes for
|
||||
// them (internal/broker): a module's own state is the read grants of its bucket and `$KV.<bucket>.>` to write it
|
||||
// (state.go stateGrants, readGrants), a read of another module's bucket is the read grants alone, and a module that uses
|
||||
// the seat operator-channel submits its own `accept.ask.<module>`, `accept.cancel.<module>` and `accept.condition.<module>`,
|
||||
// hears its own `event.decided.<module>` and reads `$KV.asks.<module>.>` of the router's record (seattraffic.go
|
||||
// SeatTrafficOf, with the messenger's seat declaring accepts ask, cancel, condition, emits decided, by-caller and records
|
||||
// asks). The gate is a bundle on the tool runner's account (ADR 0437 §3), so the gate's writes are the account node-tools'.
|
||||
// The permissions are these lists written out, not the controller's function: the broker tests of issue 1243 prove that.
|
||||
func readOf(bucket string) []string {
|
||||
stream := "KV_" + bucket
|
||||
return []string{"$JS.API.STREAM.INFO." + stream, "$JS.API.DIRECT.GET." + stream + ".>", "$JS.API.CONSUMER.CREATE." + stream + ".>",
|
||||
"$JS.API.CONSUMER.DELETE." + stream + ".>", "$JS.FC." + stream + ".>"}
|
||||
}
|
||||
|
||||
func keepsOf(bucket string) []string { return append(readOf(bucket), "$KV."+bucket+".>") }
|
||||
|
||||
func user(name string, pub, sub []string) string {
|
||||
q := func(l []string) string {
|
||||
var b strings.Builder
|
||||
for _, s := range l {
|
||||
b.WriteString(fmt.Sprintf(" %q\n", s))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
return fmt.Sprintf(` { user: %q, password: "pw"
|
||||
permissions {
|
||||
publish { allow: [
|
||||
%s ] }
|
||||
subscribe { allow: [
|
||||
%s ] }
|
||||
allow_responses: true
|
||||
}
|
||||
}
|
||||
`, name, q(pub), q(append([]string{"_INBOX." + name + ".>"}, sub...)))
|
||||
}
|
||||
|
||||
const (
|
||||
tellerAnswers = "mesh-delivery-teller_answers"
|
||||
tellerAsked = "mesh-delivery-teller_asked"
|
||||
gateAsksKV = "mesh-delivery-gate_asks"
|
||||
seatAsk = "mesh.seat.operator-channel.accept.ask.mesh-delivery-teller"
|
||||
seatCancel = "mesh.seat.operator-channel.accept.cancel.mesh-delivery-teller"
|
||||
seatCondition = "mesh.seat.operator-channel.accept.condition.mesh-delivery-teller"
|
||||
seatDecided = "mesh.seat.operator-channel.event.decided.mesh-delivery-teller"
|
||||
)
|
||||
|
||||
func fullConf(dir string) string {
|
||||
teller := append(append(append(keepsOf(tellerAnswers), keepsOf(tellerAsked)...), readOf(gateAsksKV)...),
|
||||
seatAsk, seatCancel, seatCondition, "$JS.API.DIRECT.GET.KV_asks.$KV.asks.mesh-delivery-teller.>")
|
||||
view := append(readOf(tellerAnswers), readOf(gateAsksKV)...)
|
||||
tools := append(keepsOf(gateAsksKV), readOf(tellerAnswers)...)
|
||||
return fmt.Sprintf(`
|
||||
listen: 127.0.0.1:-1
|
||||
jetstream { store_dir: %q }
|
||||
accounts {
|
||||
MESH {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "ctl", password: "pw" }
|
||||
%s%s%s ]
|
||||
}
|
||||
}
|
||||
`, filepath.Join(dir, "js"),
|
||||
user("n1.mesh-delivery-teller", teller, []string{seatDecided}),
|
||||
user("n1.mesh-delivery-view", view, nil),
|
||||
user("n1.node-tools", tools, nil))
|
||||
}
|
||||
|
||||
type fullBed struct {
|
||||
srv *server.Server
|
||||
ctl *nats.Conn
|
||||
js jetstream.JetStream
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// startFull starts the bus with the principals above and what the controller makes before any module starts: the buckets
|
||||
// of the gate's asks, the teller's answers and asked, the router's work queue SEAT_OPERATOR_CHANNEL.
|
||||
func startFull(t *testing.T) *fullBed {
|
||||
t.Helper()
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "t")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
conf := filepath.Join(dir, "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte(fullConf(dir)), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
srv, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(ctl.Close)
|
||||
js, err := jetstream.New(ctl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
for name, history := range map[string]uint8{gateAsksKV: delivery.AsksHistory, tellerAnswers: delivery.AnswersHistory, tellerAsked: 1} {
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "SEAT_OPERATOR_CHANNEL", Subjects: []string{"mesh.seat.operator-channel.accept.>"},
|
||||
Retention: jetstream.WorkQueuePolicy}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &fullBed{srv: srv, ctl: ctl, js: js, ctx: ctx}
|
||||
}
|
||||
|
||||
func (b *fullBed) as(t *testing.T, name string) jetstream.JetStream {
|
||||
t.Helper()
|
||||
c, err := nats.Connect(b.srv.ClientURL(), nats.UserInfo(name, "pw"), nats.CustomInboxPrefix("_INBOX."+name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(c.Close)
|
||||
js, err := jetstream.New(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return js
|
||||
}
|
||||
|
||||
// put writes a key as a principal, within a bounded time: a write the permissions refuse reaches the writer as a timeout.
|
||||
func (b *fullBed) put(t *testing.T, js jetstream.JetStream, bucket, key string) error {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(b.ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
kv, err := js.KeyValue(ctx, bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("%s cannot be bound: %v", bucket, err)
|
||||
}
|
||||
_, err = kv.Put(ctx, key, []byte(`{}`))
|
||||
return err
|
||||
}
|
||||
|
||||
// TestCredentialsWriteOnlyTheirOwnBuckets: the teller's credential is refused writing the gate's asks, and the gate's
|
||||
// (the tool runner's) and the view's are refused writing the teller's answers; each writes where it may, so a refusal is the
|
||||
// permissions' and not a bus that does not take writes (1343/AC4, ADR 0437 §6's check).
|
||||
func TestCredentialsWriteOnlyTheirOwnBuckets(t *testing.T) { // 1343/AC4
|
||||
b := startFull(t)
|
||||
teller, view, tools := b.as(t, "n1.mesh-delivery-teller"), b.as(t, "n1.mesh-delivery-view"), b.as(t, "n1.node-tools")
|
||||
if err := b.put(t, teller, tellerAnswers, "merge.g.0123456789ab"); err != nil {
|
||||
t.Errorf("the teller cannot write its own answers: %v", err)
|
||||
}
|
||||
if err := b.put(t, tools, gateAsksKV, "merge.g.0123456789ab"); err != nil {
|
||||
t.Errorf("the gate's account cannot write the gate's asks: %v", err)
|
||||
}
|
||||
if err := b.put(t, teller, gateAsksKV, "merge.g.ffffffffffff"); err == nil {
|
||||
t.Error("the teller's credential wrote the gate's asks")
|
||||
}
|
||||
if err := b.put(t, tools, tellerAnswers, "merge.g.ffffffffffff"); err == nil {
|
||||
t.Error("the gate's account wrote the teller's answers")
|
||||
}
|
||||
if err := b.put(t, view, tellerAnswers, "merge.g.ffffffffffff"); err == nil {
|
||||
t.Error("the view's credential wrote the teller's answers")
|
||||
}
|
||||
// What was refused is not there.
|
||||
for _, c := range []struct{ bucket, key string }{{gateAsksKV, "merge.g.ffffffffffff"}, {tellerAnswers, "merge.g.ffffffffffff"}} {
|
||||
kv, _ := b.js.KeyValue(b.ctx, c.bucket)
|
||||
if _, err := kv.Get(b.ctx, c.key); err == nil {
|
||||
t.Errorf("%s holds %s though its write was refused", c.bucket, c.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestTellerCarriesAnAskAndWritesTheAnswer starts the program on the composed principal against the bus: the gate's account
|
||||
// writes an ask, the teller publishes it to the seat's work queue under its own name, the router (the test) says the
|
||||
// warrant on the teller's decided subject, and the answer is under the ask's key in the teller's bucket; withdrawing a
|
||||
// second ask cancels it (1343/AC1, AC2).
|
||||
func TestTellerCarriesAnAskAndWritesTheAnswer(t *testing.T) { // 1343/AC1, AC2
|
||||
now = time.Now().UTC()
|
||||
b := startFull(t)
|
||||
tools := b.as(t, "n1.node-tools")
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||
logs := &syncBuffer{}
|
||||
rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) })
|
||||
t.Cleanup(func() { rn.finish(t, 10*time.Second) })
|
||||
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
kv, err := tools.KeyValue(b.ctx, gateAsksKV)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The program hears warrants before it reads the asks; give it its subscription before the ask is written.
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
if _, err := kv.Put(b.ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seat, err := b.js.Stream(b.ctx, "SEAT_OPERATOR_CHANNEL")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var raw *jetstream.RawStreamMsg
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if raw == nil {
|
||||
t.Fatalf("no ask on the seat's queue: %v\n%s", err, logs.String())
|
||||
}
|
||||
var asked asks.Ask
|
||||
if err := json.Unmarshal(raw.Data, &asked); err != nil || asked.Digest() != ga.Ask.Digest() {
|
||||
t.Fatalf("the ask on the queue is %+v (%v), want the gate's", asked, err)
|
||||
}
|
||||
// The router says the warrant.
|
||||
w, _ := json.Marshal(warrantFor(ga.Ask, "approve"))
|
||||
if err := b.ctl.Publish(seatDecided, w); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
answers, err := b.js.KeyValue(b.ctx, tellerAnswers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var e jetstream.KeyValueEntry
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err = answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if e == nil {
|
||||
t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String())
|
||||
}
|
||||
if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "approve" {
|
||||
t.Errorf("answer %+v err %v", a, err)
|
||||
}
|
||||
// A second ask, withdrawn by the gate, is cancelled at the router.
|
||||
other := []delivery.Head{{Repository: "novox/mesh-delivery", Number: 13, Head: headB}}
|
||||
key2, value2, ga2 := gateAsk(t, other, other)
|
||||
if _, err := kv.Put(b.ctx, key2, value2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if _, err := seat.GetLastMsgForSubject(b.ctx, seatAsk); err == nil && strings.Contains(logs.String(), ga2.Ask.ID) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := kv.Delete(b.ctx, key2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if raw, err = seat.GetLastMsgForSubject(b.ctx, seatCancel); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil || string(raw.Data) != `{"id":"`+ga2.Ask.ID+`"}` {
|
||||
t.Errorf("the withdrawn ask was not cancelled: %v %v\n%s", raw, err, logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// kvStore is a state bucket of the teller as the asker's store (mesh-sdk go/asker Store): a compare-and-set that lost is
|
||||
// stdio.ErrChanged, as the runtime's state reports it.
|
||||
type kvStore struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (s kvStore) Get(key string) (*stdio.StateEntry, error) {
|
||||
e, err := s.kv.Get(s.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &stdio.StateEntry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil
|
||||
}
|
||||
|
||||
func (s kvStore) Create(key string, value any) (uint64, error) {
|
||||
b, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := s.kv.Create(s.ctx, key, b)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (s kvStore) Update(key string, value any, revision uint64) (uint64, error) {
|
||||
b, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rev, err := s.kv.Update(s.ctx, key, b, revision)
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return 0, fmt.Errorf("%s: %w", key, stdio.ErrChanged)
|
||||
}
|
||||
return rev, err
|
||||
}
|
||||
|
||||
func (s kvStore) Delete(key string) error {
|
||||
err := s.kv.Delete(s.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// kvAnswers is the bucket answers: the teller's, written by compare-and-set: created when the key has no answer, updated at the revision read when it holds an earlier asking's.
|
||||
type kvAnswers struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (a kvAnswers) Get(key string) ([]byte, uint64, bool, error) {
|
||||
e, err := a.kv.Get(a.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
|
||||
func (a kvAnswers) Write(key string, value []byte, rev uint64, found bool) (bool, error) {
|
||||
var err error
|
||||
if found {
|
||||
_, err = a.kv.Update(a.ctx, key, value, rev)
|
||||
} else {
|
||||
_, err = a.kv.Create(a.ctx, key, value)
|
||||
}
|
||||
if errors.Is(err, jetstream.ErrKeyExists) {
|
||||
return false, nil
|
||||
}
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// kvGate reads the gate's bucket asks, which the teller may watch and read and never write.
|
||||
type kvGate struct {
|
||||
ctx context.Context
|
||||
kv jetstream.KeyValue
|
||||
}
|
||||
|
||||
func (g kvGate) Get(key string) ([]byte, bool, error) {
|
||||
e, err := g.kv.Get(g.ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return e.Value(), true, nil
|
||||
}
|
||||
@@ -1,8 +1,9 @@
|
||||
// Command mesh-delivery-teller is delivery's teller (novox/hq ADR 0437 §4, issue 1240). It will carry the gate's asks to
|
||||
// the operator and write the answers back (release 3d). In release 3c nothing writes asks, so it only runs: it connects
|
||||
// as itself, says once that it waits for the gate's asks, and writes and publishes nothing. It does not open the asks
|
||||
// bucket, for waiting for an absent bucket needs a timer (ADR 0437 §8 lets the parts hold none); the watch comes with the
|
||||
// gate. It serves no request subject.
|
||||
// Command mesh-delivery-teller is delivery's teller (novox/hq ADR 0437 §4, §6, §9; issues 1240, 1343). It carries the
|
||||
// gate's merge asks to the operator and writes the answers back: it watches the gate's asks, asks each of the router as
|
||||
// itself, and on the router's warrant, once it holds, writes the answer under the ask's key. Where the gate's asks
|
||||
// bucket does not exist yet it says once that it waits for the gate's asks and does nothing: waiting for an absent bucket
|
||||
// needs a timer (ADR 0437 §8 lets the parts hold none), so it is started again with the gate. It serves no request
|
||||
// subject.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -22,6 +23,10 @@ import (
|
||||
"syscall"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -103,10 +108,43 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return fmt.Errorf("connecting to the bus: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
logger.Print(waiting)
|
||||
<-ctx.Done()
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gate, err := js.KeyValue(ctx, delivery.AsksBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
logger.Print(waiting)
|
||||
<-ctx.Done()
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", delivery.AsksBucket, err)
|
||||
}
|
||||
var kvs [2]jetstream.KeyValue
|
||||
for i, name := range []string{delivery.AnswersBucket, askedBucket} {
|
||||
if kvs[i], err = js.KeyValue(ctx, name); err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
state := kvStore{ctx: ctx, kv: kvs[1]}
|
||||
client := &asker.Client{Module: delivery.TellerModule, State: state, Publish: publishVia(ctx, js),
|
||||
Record: recordVia(ctx, conn, delivery.TellerModule), Subscribe: decidedVia(conn, delivery.TellerModule, logger.Printf)}
|
||||
t := &Teller{Asker: client, Store: state, Answers: kvAnswers{ctx: ctx, kv: kvs[0]}, Gate: kvGate{ctx: ctx, kv: gate},
|
||||
Say: sayVia(ctx, js, delivery.TellerModule), Logf: logger.Printf}
|
||||
// Hear the warrants first, so none that comes while the asks are read is missed.
|
||||
if err := client.Listen(t.Answer, t.Said); err != nil {
|
||||
return fmt.Errorf("hearing the router's warrants: %w", err)
|
||||
}
|
||||
if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone); err != nil {
|
||||
return fmt.Errorf("watching the gate's asks: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// askedBucket is the teller's state bucket asked, `<module>_<name>`: what it asked of the router and which condition it
|
||||
// raised (manifest state entry asked, novox/hq ADR 0437 §4).
|
||||
const askedBucket = delivery.TellerModule + "_asked"
|
||||
|
||||
// waiting is said once at start.
|
||||
const waiting = "waiting for the gate's asks, which come with the gate in release 3d; nothing is read, written or published until then"
|
||||
const waiting = "waiting for the gate's asks, whose bucket does not exist yet; the teller is started again with the gate, and nothing is read, written or published until then"
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The teller carries the gate's merge asks to the operator and writes the operator's answer back (novox/hq ADR 0437
|
||||
// §6, §9). It reads the gate's asks, publishes each to the router unchanged as the asker mesh-delivery-teller
|
||||
// (mesh-sdk go/asker), withdraws it when the gate takes its key away, and on the warrant the router says to it
|
||||
// holds the warrant to the ask (ADR 0259 §6) and writes the answer under the ask's own key by compare-and-set. A
|
||||
// warrant that does not hold writes no answer and is said as a condition. It holds no timer and serves no request:
|
||||
// every reaction is a revision of the gate's key or a warrant.
|
||||
|
||||
// store is where the teller keeps what it asked: the asker's own records (`ask.<id>`) and, beside them, which gate key
|
||||
// each ask was made for and at which revision of it (`key.<id>`), and which conditions it has raised (`cond.<id>`).
|
||||
type store interface {
|
||||
asker.Store
|
||||
Delete(key string) error
|
||||
}
|
||||
|
||||
// routerAsks is the asker.Client as the teller uses it.
|
||||
type routerAsks interface {
|
||||
Ask(a asks.Ask) error
|
||||
Cancel(id string) error
|
||||
Where(id string) (state string, w *asks.Warrant, err error)
|
||||
Take(w asks.Warrant, act func(asks.Option, asks.Warrant) error) (asker.Handled, error)
|
||||
}
|
||||
|
||||
// answers is the teller's bucket answers.
|
||||
type answers interface {
|
||||
// Get is the answer under the ask's key and its revision; found is false when there is none.
|
||||
Get(key string) (value []byte, rev uint64, found bool, err error)
|
||||
// Write creates the answer when found is false (the key had none when it was read) and otherwise updates it at the
|
||||
// revision read; written is false when another writer was first.
|
||||
Write(key string, value []byte, rev uint64, found bool) (written bool, err error)
|
||||
}
|
||||
|
||||
// gateAsks reads the gate's bucket asks.
|
||||
type gateAsks interface {
|
||||
Get(key string) (value []byte, found bool, err error)
|
||||
}
|
||||
|
||||
// Teller is the teller's logic; the program's wiring is main.go and decided.go.
|
||||
type Teller struct {
|
||||
Asker routerAsks
|
||||
Store store
|
||||
Answers answers
|
||||
Gate gateAsks
|
||||
// Say publishes a condition to the operator (mesh.seat.operator-channel.accept.condition.<module>).
|
||||
Say func(c asks.Condition) error
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (t *Teller) logf(format string, args ...any) {
|
||||
if t.Logf != nil {
|
||||
t.Logf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// seen is what the teller keeps beside an ask it made. Under `key.<router id>` it names the gate key the asking was made for;
|
||||
// under `gate.<gate key>` it names the router id of the latest asking of that key and the revision of the key it was made
|
||||
// at, so a replay of the revision asks nothing again and an overwrite with a new id is known for a new asking.
|
||||
type seen struct {
|
||||
Key string `json:"key,omitempty"`
|
||||
ID string `json:"id,omitempty"`
|
||||
Rev uint64 `json:"rev"`
|
||||
}
|
||||
|
||||
func gateRecordKey(key string) string { return "gate." + key }
|
||||
func recordKey(id string) string { return "key." + id }
|
||||
func keptKey(id string) string { return "ask." + id }
|
||||
func condKey(id string) string { return "cond." + id }
|
||||
|
||||
// condID is what a condition of an ask is kept and named by: the gate key it was asked for, so the asking that follows a bad
|
||||
// answer clears the condition of the one before it. An id the teller holds no key for is its own.
|
||||
func (t *Teller) condID(id string) string {
|
||||
if s, e, _ := t.seenOf(id); e != nil && s.Key != "" {
|
||||
return s.Key
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func (t *Teller) seenAt(rk string) (seen, *stdio.StateEntry, error) {
|
||||
e, err := t.Store.Get(rk)
|
||||
if err != nil || e == nil {
|
||||
return seen{}, nil, err
|
||||
}
|
||||
var s seen
|
||||
if err := json.Unmarshal(e.Value, &s); err != nil {
|
||||
return seen{}, nil, fmt.Errorf("the record %s cannot be read: %w", rk, err)
|
||||
}
|
||||
return s, e, nil
|
||||
}
|
||||
|
||||
// seenOf is the record of a router id: the gate key it was asked for.
|
||||
func (t *Teller) seenOf(id string) (seen, *stdio.StateEntry, error) { return t.seenAt(recordKey(id)) }
|
||||
|
||||
func (t *Teller) keptOf(id string) (*asker.Kept, uint64, error) {
|
||||
e, err := t.Store.Get(keptKey(id))
|
||||
if err != nil || e == nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
var k asker.Kept
|
||||
if err := json.Unmarshal(e.Value, &k); err != nil {
|
||||
return nil, 0, fmt.Errorf("the record %s cannot be read: %w", keptKey(id), err)
|
||||
}
|
||||
return &k, e.Revision, nil
|
||||
}
|
||||
|
||||
// open says whether an ask is made and neither answered nor ended.
|
||||
func open(k *asker.Kept) bool { return k != nil && !k.Acted && k.Ended == "" }
|
||||
|
||||
// note keeps that the asking with this router id is the latest of the gate key, made at revision rev of it.
|
||||
func (t *Teller) note(id, key string, rev uint64, prior *stdio.StateEntry) error {
|
||||
if _, err := t.Store.Create(recordKey(id), seen{Key: key, Rev: rev}); err != nil && !errors.Is(err, stdio.ErrChanged) {
|
||||
return err
|
||||
}
|
||||
g := seen{ID: id, Rev: rev}
|
||||
if prior == nil {
|
||||
_, err := t.Store.Create(gateRecordKey(key), g)
|
||||
return err
|
||||
}
|
||||
_, err := t.Store.Update(gateRecordKey(key), g, prior.Revision)
|
||||
return err
|
||||
}
|
||||
|
||||
// OnAsk handles the gate's asks key put at revision rev. Each asking has its own router id, which is the value's ask.id and
|
||||
// never one recomputed from the key (mesh-sdk go/delivery Ask.Asking, AskingID): the router keeps an ended ask and drops a
|
||||
// second ask under its id. So the ask is published once per id; a key overwritten at a higher revision under a new id is a
|
||||
// new asking, and the earlier asking's router ask is cancelled first. A key whose value is not the ask its name says, one
|
||||
// already answered, and a revision the teller has handled are left. An ask still open and unchanged is only noted (and its
|
||||
// warrant read from the router, in case it came while the teller was away). A value that changed its words under the id
|
||||
// it was asked under is cancelled and not asked again: the router would drop it, so the gate must give it a new asking.
|
||||
func (t *Teller) OnAsk(key string, rev uint64, value []byte) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
ga, err := delivery.DecodeAsk(value)
|
||||
if err != nil {
|
||||
t.logf("the gate's key %s at revision %d is not an ask, and is left: %v", key, rev, err)
|
||||
return
|
||||
}
|
||||
if want, err := delivery.MergeAskKey(ga.Group, ga.Heads); err != nil || want != key {
|
||||
t.logf("the gate's key %s at revision %d holds the ask of %q, not its own, and is left (%v)", key, rev, want, err)
|
||||
return
|
||||
}
|
||||
id := ga.Ask.ID
|
||||
if raw, _, has, err := t.Answers.Get(key); err != nil {
|
||||
t.logf("%s: the answers could not be read, so the ask is left: %v", key, err)
|
||||
return
|
||||
} else if has && answeredAsking(raw) == id {
|
||||
return // this asking is answered; an earlier asking's answer does not stop a later asking
|
||||
}
|
||||
prior, grec, err := t.seenAt(gateRecordKey(key))
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
return
|
||||
}
|
||||
if grec != nil && rev <= prior.Rev {
|
||||
if k, _, err := t.keptOf(prior.ID); err == nil && open(k) {
|
||||
t.catchUp(prior.ID)
|
||||
}
|
||||
return
|
||||
}
|
||||
if grec != nil && prior.ID != id {
|
||||
// A new asking of the group at these heads: the earlier one is taken back.
|
||||
t.cancelOpen(key, prior.ID)
|
||||
}
|
||||
k, _, err := t.keptOf(id)
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
return
|
||||
}
|
||||
if k != nil {
|
||||
switch {
|
||||
case open(k) && k.Ask.Digest() == ga.Ask.Digest():
|
||||
_ = t.note(id, key, rev, grec)
|
||||
t.catchUp(id)
|
||||
case open(k):
|
||||
t.cancelOpen(key, id)
|
||||
_ = t.note(id, key, rev, grec)
|
||||
t.logf("%s: the ask %s changed its words at revision %d under the same id: cancelled, and not asked again, for the router drops a second ask under an id (the gate gives a new asking a new id)", key, id, rev)
|
||||
default:
|
||||
_ = t.note(id, key, rev, grec)
|
||||
t.logf("%s: the ask %s ended, and the gate wrote it again under the same id at revision %d: not asked again, the router would drop it", key, id, rev)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := t.note(id, key, rev, grec); err != nil {
|
||||
t.logf("%s: the record of the ask %s could not be kept, so it is left: %v", key, id, err)
|
||||
return
|
||||
}
|
||||
if err := t.Asker.Ask(ga.Ask); err != nil {
|
||||
t.logf("%s at revision %d: the ask %s was not asked: %v", key, rev, id, err)
|
||||
if kept, _, _ := t.keptOf(id); open(kept) {
|
||||
// Kept but not published: forgotten, so the next revision of the key asks it.
|
||||
_ = t.Store.Delete(keptKey(id))
|
||||
}
|
||||
return
|
||||
}
|
||||
t.logf("%s at revision %d: the ask %s is asked of the operator (asking %d)", key, rev, id, ga.Asking)
|
||||
}
|
||||
|
||||
// cancelOpen takes an open ask back at the router and marks it ended here, since the router's word that it ended may come
|
||||
// later. An ask that is not open is left.
|
||||
func (t *Teller) cancelOpen(key, id string) {
|
||||
k, krev, err := t.keptOf(id)
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
return
|
||||
}
|
||||
if !open(k) {
|
||||
return
|
||||
}
|
||||
if err := t.Asker.Cancel(id); err != nil {
|
||||
t.logf("%s: the ask %s could not be cancelled: %v", key, id, err)
|
||||
return
|
||||
}
|
||||
k.Ended = asks.OutcomeCancelled
|
||||
if _, err := t.Store.Update(keptKey(id), k, krev); err != nil && !errors.Is(err, stdio.ErrChanged) {
|
||||
t.logf("%s: the cancelled ask %s could not be marked: %v", key, id, err)
|
||||
}
|
||||
t.logf("%s: the ask %s is cancelled", key, id)
|
||||
}
|
||||
|
||||
// OnAskGone handles the gate's asks key taken away (a withdrawn ask): the latest asking's router ask is cancelled, and an
|
||||
// answer to it merges nothing.
|
||||
func (t *Teller) OnAskGone(key string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if _, _, _, err := delivery.ParseAskKey(key); err != nil {
|
||||
t.logf("the gate's key %s was taken away and is no ask key, and is left: %v", key, err)
|
||||
return
|
||||
}
|
||||
prior, grec, err := t.seenAt(gateRecordKey(key))
|
||||
if err != nil || grec == nil {
|
||||
if err != nil {
|
||||
t.logf("%s: %v", key, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.cancelOpen(key, prior.ID)
|
||||
}
|
||||
|
||||
// catchUp reads the router's record of an ask the teller made and hands its warrant on, as the router's event would:
|
||||
// a warrant that came while the teller was away.
|
||||
func (t *Teller) catchUp(id string) {
|
||||
state, w, err := t.Asker.Where(id)
|
||||
if err != nil {
|
||||
t.logf("the router's record of the ask %s could not be read: %v", id, err)
|
||||
return
|
||||
}
|
||||
if w == nil {
|
||||
return
|
||||
}
|
||||
t.logf("the ask %s is %s at the router: its warrant is handled now", id, state)
|
||||
h, err := t.Asker.Take(*w, t.Answer)
|
||||
t.Said(h, *w, err)
|
||||
}
|
||||
|
||||
// mergeAct is what the approve option of the gate's ask binds: this ask, the answer, and the heads it covers. It is
|
||||
// spelled as the gate composes it (mesh-catalog modules/mesh-delivery-gate approval.go mergeAct, from
|
||||
// modules/mesh-delivery approval.go:63-65) and as mesh-controller verifies it (cmd/mesh-controller/moved.go:98-106).
|
||||
func mergeAct(ask, option string, heads map[string]string) asks.Act {
|
||||
var out []string
|
||||
for id, sha := range heads {
|
||||
out = append(out, id+"="+sha)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return asks.Act{"verb": "mesh-delivery.merge", "ask": ask, "option": option, "heads": strings.Join(out, ",")}
|
||||
}
|
||||
|
||||
// headsOf is the heads an ask covers as an act names them: a delivery's id, `owner/repository@<twelve characters of its
|
||||
// commit>` with the repository in lower case, as the gate's IDOf spells it (mesh-catalog modules/mesh-delivery-gate model.go),
|
||||
// and its whole commit.
|
||||
func headsOf(heads []delivery.Head) map[string]string {
|
||||
out := map[string]string{}
|
||||
for _, h := range heads {
|
||||
short := h.Head
|
||||
if len(short) > 12 {
|
||||
short = short[:12]
|
||||
}
|
||||
out[strings.ToLower(h.Repository)+"@"+short] = h.Head
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// answeredAsking is the router id of the asking an answer is for: its warrant's id before the "@". An answer that cannot be
|
||||
// read answers no asking.
|
||||
func answeredAsking(raw []byte) string {
|
||||
a, err := delivery.DecodeAnswer(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
id, _, _ := strings.Cut(a.Warrant, "@")
|
||||
return id
|
||||
}
|
||||
|
||||
// warrantID names a warrant, which carries no id of its own: the ask and the time the router gave it.
|
||||
func warrantID(w asks.Warrant) string {
|
||||
return w.Ask + "@" + w.At.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// Answer is the act of a warrant that holds the ask (asker.Client.Take has checked its asker, the ask's digest, the
|
||||
// option and its level, and the time): the option's bound act must be the act of this ask and the heads it covers,
|
||||
// and then the answer is written under the ask's key, once. An error is a warrant that does not hold here.
|
||||
func (t *Teller) Answer(o asks.Option, w asks.Warrant) error {
|
||||
s, srec, err := t.seenOf(w.Ask)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if srec == nil {
|
||||
return fmt.Errorf("the teller holds no gate key for the ask %s", w.Ask)
|
||||
}
|
||||
raw, found, err := t.Gate.Get(s.Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the gate's ask %s could not be read: %w", s.Key, err)
|
||||
}
|
||||
if !found {
|
||||
t.logf("%s: the ask %s was answered and the gate has taken it away: an answer to a withdrawn ask merges nothing", s.Key, w.Ask)
|
||||
return nil
|
||||
}
|
||||
ga, err := delivery.DecodeAsk(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the gate's ask %s is not an ask: %w", s.Key, err)
|
||||
}
|
||||
if ga.Ask.ID != w.Ask {
|
||||
t.logf("%s: the ask %s was answered, and the gate has asked again under %s: an answer to an earlier asking merges nothing", s.Key, w.Ask, ga.Ask.ID)
|
||||
return nil
|
||||
}
|
||||
if ga.Ask.Digest() != w.AskDigest {
|
||||
return fmt.Errorf("the gate's ask %s is no longer the ask the operator answered (its digest is %s, the warrant's %s)",
|
||||
s.Key, ga.Ask.Digest(), w.AskDigest)
|
||||
}
|
||||
if err := o.Performs(mergeAct(w.Ask, o.ID, headsOf(ga.Heads))); err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := delivery.EncodeAnswer(delivery.Answer{Option: o.ID, Warrant: warrantID(w), Words: w.Says()})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Compare-and-set: created when the key has no answer; updated at the revision read when it holds an earlier asking's
|
||||
// answer (the gate ignores that one); left when it holds this asking's.
|
||||
akey := delivery.AnswerKey(s.Key)
|
||||
cur, crev, found, err := t.Answers.Get(akey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the answer under %s could not be read: %w", s.Key, err)
|
||||
}
|
||||
if found && answeredAsking(cur) == w.Ask {
|
||||
t.logf("%s: an answer is there already, and is not written again", s.Key)
|
||||
return nil
|
||||
}
|
||||
written, err := t.Answers.Write(akey, body, crev, found)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the answer under %s could not be written: %w", s.Key, err)
|
||||
}
|
||||
if !written {
|
||||
t.logf("%s: another writer answered first, and the answer is not written again", s.Key)
|
||||
return nil
|
||||
}
|
||||
t.logf("%s: %s", s.Key, w.Says())
|
||||
return nil
|
||||
}
|
||||
|
||||
// Said is what the teller does with what became of a warrant: a good answer clears the condition of its ask; a warrant
|
||||
// that does not hold, or that the router refused, is said as a condition (ADR 0437 §9). Nothing is written for either.
|
||||
func (t *Teller) Said(h asker.Handled, w asks.Warrant, err error) {
|
||||
switch {
|
||||
case h == asker.Acted && err == nil:
|
||||
t.logf("the warrant of %s was %s", w.Ask, h)
|
||||
t.clear(w.Ask)
|
||||
case h == asker.Refused || (h == asker.Acted && err != nil):
|
||||
t.logf("the warrant of %s does not hold, and no answer is written: %v", w.Ask, err)
|
||||
t.raise(w.Ask, "A merge Approve did not hold", "its answer did not hold here: "+oneLine(err))
|
||||
case h == asker.Recorded && w.Outcome == asks.OutcomeRefused:
|
||||
t.logf("the router refused the ask %s: %s", w.Ask, w.Words)
|
||||
t.raise(w.Ask, "A merge ask was refused", "the router refused it: "+w.Words)
|
||||
case err != nil:
|
||||
t.logf("the warrant of %s: %v", w.Ask, err)
|
||||
}
|
||||
}
|
||||
|
||||
func oneLine(err error) string {
|
||||
if err == nil {
|
||||
return "no reason was given"
|
||||
}
|
||||
s := strings.Join(strings.Fields(err.Error()), " ")
|
||||
if r := []rune(s); len(r) > 200 {
|
||||
return string(r[:199]) + "…"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// conditionKey is the key of the condition about an ask.
|
||||
func conditionKey(id string) string { return delivery.TellerModule + ".merge-ask." + id }
|
||||
|
||||
func (t *Teller) raise(askID, headline, why string) {
|
||||
if t.Say == nil {
|
||||
return
|
||||
}
|
||||
id := t.condID(askID)
|
||||
if _, err := t.Store.Create(condKey(id), seen{}); err != nil {
|
||||
if errors.Is(err, stdio.ErrChanged) {
|
||||
return // raised already
|
||||
}
|
||||
t.logf("the condition of %s could not be kept: %v", id, err)
|
||||
}
|
||||
needs := "to ask for the Approve again"
|
||||
c := asks.Condition{Key: conditionKey(id), Severity: asks.SeverityWarning, Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateRaised,
|
||||
Summary: "The merge ask " + id + ": " + why, Headline: headline, Needs: needs,
|
||||
Explanation: asks.NeedsYou + needs + ". The merge ask " + id + ", " + why + ". Nothing was merged.",
|
||||
Resolved: "A good answer to the merge ask was written."}
|
||||
if err := t.Say(c); err != nil {
|
||||
t.logf("the condition of %s could not be said: %v", id, err)
|
||||
_ = t.Store.Delete(condKey(id))
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Teller) clear(askID string) {
|
||||
if t.Say == nil {
|
||||
return
|
||||
}
|
||||
id := t.condID(askID)
|
||||
e, err := t.Store.Get(condKey(id))
|
||||
if err != nil || e == nil {
|
||||
return
|
||||
}
|
||||
c := asks.Condition{Key: conditionKey(id), Subject: asks.Scope{Scope: "merge", ID: id}, State: asks.StateCleared}
|
||||
if err := t.Say(c); err != nil {
|
||||
t.logf("the condition of %s could not be cleared: %v", id, err)
|
||||
return
|
||||
}
|
||||
_ = t.Store.Delete(condKey(id))
|
||||
}
|
||||
@@ -0,0 +1,505 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/asker"
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const (
|
||||
headA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
)
|
||||
|
||||
var now = time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)
|
||||
|
||||
// memStore is a state bucket in memory with the revisions and the compare-and-set of the runtime's state.
|
||||
type memStore struct {
|
||||
entries map[string]*stdio.StateEntry
|
||||
seq uint64
|
||||
}
|
||||
|
||||
func (m *memStore) Get(key string) (*stdio.StateEntry, error) {
|
||||
if e, ok := m.entries[key]; ok {
|
||||
c := *e
|
||||
return &c, nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *memStore) put(key string, value any) (uint64, error) {
|
||||
b, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
m.seq++
|
||||
m.entries[key] = &stdio.StateEntry{Key: key, Value: b, Revision: m.seq}
|
||||
return m.seq, nil
|
||||
}
|
||||
|
||||
func (m *memStore) Create(key string, value any) (uint64, error) {
|
||||
if _, ok := m.entries[key]; ok {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
return m.put(key, value)
|
||||
}
|
||||
|
||||
func (m *memStore) Update(key string, value any, revision uint64) (uint64, error) {
|
||||
if e, ok := m.entries[key]; !ok || e.Revision != revision {
|
||||
return 0, stdio.ErrChanged
|
||||
}
|
||||
return m.put(key, value)
|
||||
}
|
||||
|
||||
func (m *memStore) Delete(key string) error { delete(m.entries, key); return nil }
|
||||
|
||||
type memAnswers struct {
|
||||
written map[string][]byte
|
||||
revs map[string]uint64
|
||||
seq uint64
|
||||
writes int
|
||||
}
|
||||
|
||||
func (a *memAnswers) Get(key string) ([]byte, uint64, bool, error) {
|
||||
v, ok := a.written[key]
|
||||
return v, a.revs[key], ok, nil
|
||||
}
|
||||
|
||||
// Write is the bucket's compare-and-set: a create of a key that has an answer, or an update at another revision, loses.
|
||||
func (a *memAnswers) Write(key string, value []byte, rev uint64, found bool) (bool, error) {
|
||||
_, has := a.written[key]
|
||||
if found != has || (found && rev != a.revs[key]) {
|
||||
return false, nil
|
||||
}
|
||||
a.seq++
|
||||
a.writes++
|
||||
if a.revs == nil {
|
||||
a.revs = map[string]uint64{}
|
||||
}
|
||||
a.written[key], a.revs[key] = value, a.seq
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type memGate map[string][]byte
|
||||
|
||||
func (g memGate) Get(key string) ([]byte, bool, error) { v, ok := g[key]; return v, ok, nil }
|
||||
|
||||
type published struct {
|
||||
subject string
|
||||
body json.RawMessage
|
||||
id string
|
||||
}
|
||||
|
||||
// bed is a teller with the asker.Client of the SDK on in-memory ends: what is published, the router's records, the
|
||||
// warrants it would hear, and the conditions it says.
|
||||
type bed struct {
|
||||
t *Teller
|
||||
client *asker.Client
|
||||
store *memStore
|
||||
answers *memAnswers
|
||||
gate memGate
|
||||
pubs []published
|
||||
conds []asks.Condition
|
||||
records map[string]string
|
||||
// seenIDs are the ids the fake router has taken an ask under, and dropped the asks it took a second time under: the
|
||||
// messenger keeps an ask after it ends and returns for an ask whose id it holds (mesh-catalog modules/messenger
|
||||
// cmd/messenger/asks.go:487-489, keepAsk), so a reused id reaches no phone.
|
||||
seenIDs map[string]bool
|
||||
dropped []string
|
||||
hear func(stdio.Envelope) error
|
||||
logs strings.Builder
|
||||
}
|
||||
|
||||
func newBed(t *testing.T) *bed {
|
||||
t.Helper()
|
||||
b := &bed{store: &memStore{entries: map[string]*stdio.StateEntry{}}, answers: &memAnswers{written: map[string][]byte{}, revs: map[string]uint64{}},
|
||||
gate: memGate{}, records: map[string]string{}, seenIDs: map[string]bool{}}
|
||||
b.client = &asker.Client{Module: delivery.TellerModule, State: b.store, Now: func() time.Time { return now },
|
||||
Publish: func(subject string, body any, id string) error {
|
||||
raw, err := json.Marshal(body)
|
||||
if subject == asks.AskSubject(delivery.TellerModule) {
|
||||
if b.seenIDs[id] {
|
||||
b.dropped = append(b.dropped, id)
|
||||
}
|
||||
b.seenIDs[id] = true
|
||||
}
|
||||
b.pubs = append(b.pubs, published{subject, raw, id})
|
||||
return err
|
||||
},
|
||||
Record: func(id string) (*stdio.StateEntry, error) {
|
||||
if v, ok := b.records[id]; ok {
|
||||
return &stdio.StateEntry{Key: id, Value: json.RawMessage(v)}, nil
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
Subscribe: func(_ string, handle func(stdio.Envelope) error) error { b.hear = handle; return nil },
|
||||
}
|
||||
b.t = &Teller{Asker: b.client, Store: b.store, Answers: b.answers, Gate: b.gate,
|
||||
Say: func(c asks.Condition) error {
|
||||
if _, err := asks.EncodeCondition(c); err != nil {
|
||||
t.Errorf("a condition the router would refuse: %v", err)
|
||||
}
|
||||
b.conds = append(b.conds, c)
|
||||
return nil
|
||||
},
|
||||
Logf: func(f string, a ...any) { fmt.Fprintf(&b.logs, f+"\n", a...) }}
|
||||
if err := b.client.Listen(b.t.Answer, b.t.Said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// gateAsk is an ask as the gate composes it: the router's ask with the approve and decline options binding the act
|
||||
// `mesh-delivery.merge` for this ask and the heads (mesh-catalog modules/mesh-delivery approval.go:123-191 composeMergeAsk
|
||||
// and :63-65 mergeAct), under the id its key gives (mesh-sdk go/delivery AskID). boundTo are the heads the options bind,
|
||||
// which are the ask's own unless a test says another.
|
||||
func gateAsk(t *testing.T, heads, boundTo []delivery.Head) (key string, value []byte, ask delivery.Ask) {
|
||||
t.Helper()
|
||||
return gateAskN(t, heads, boundTo, 1)
|
||||
}
|
||||
|
||||
// gateAskN is the asking-th writing of the group's ask at these heads: its router id is AskingID(key, asking) (mesh-sdk
|
||||
// go/delivery Ask.Asking), the first asking's the key's own.
|
||||
func gateAskN(t *testing.T, heads, boundTo []delivery.Head, asking int) (key string, value []byte, ask delivery.Ask) {
|
||||
t.Helper()
|
||||
key, err := delivery.MergeAskKey("g", heads)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := delivery.AskingID(key, asking)
|
||||
a := asks.Ask{ID: id, Headline: "Merge into main: novox/mesh-delivery#12", Explanation: "Merges two pull requests.\nReview: approved.",
|
||||
Who: asks.Operator, Expires: now.Add(asks.ApproveLasts - 10*time.Minute), OnExpiry: "the merge stays held", About: "delivery.mesh-delivery.12"}
|
||||
for _, o := range []struct{ id, label, does string }{{"approve", "Approve", "merges"}, {"decline", "Decline", "merges nothing"}} {
|
||||
binds, err := asks.ActDigest(mergeAct(id, o.id, headsOf(boundTo)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.Options = append(a.Options, asks.Option{ID: o.id, Label: o.label, Does: o.does, Level: asks.Approve, Binds: binds})
|
||||
}
|
||||
ask = delivery.Ask{Group: "g", Heads: heads, Asking: asking, Ask: a}
|
||||
value, err = delivery.EncodeAsk(ask)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return key, value, ask
|
||||
}
|
||||
|
||||
func twoHeads() []delivery.Head {
|
||||
return []delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: headA}, {Repository: "novox/mesh-sdk", Number: 4, Head: headB}}
|
||||
}
|
||||
|
||||
// warrantFor is the router's warrant for a chosen option of an ask, as the router verified the operator.
|
||||
func warrantFor(a asks.Ask, option string) asks.Warrant {
|
||||
return asks.Warrant{Ask: a.ID, Asker: delivery.TellerModule, Outcome: asks.OutcomeChosen, Option: option, Label: "Approve", Level: asks.Approve,
|
||||
By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id"}, Channel: "telegram",
|
||||
At: now.Add(time.Hour), AskDigest: a.Digest()}
|
||||
}
|
||||
|
||||
func (b *bed) deliver(t *testing.T, w asks.Warrant) {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(w)
|
||||
if err := b.hear(stdio.Envelope{Key: asks.Seat + ".decided." + delivery.TellerModule, Body: raw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *bed) askedOf(subject string) []published {
|
||||
var out []published
|
||||
for _, p := range b.pubs {
|
||||
if p.subject == subject {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAnAsksKeyBecomesOneRouterAskAndDeletingItCancelsIt(t *testing.T) { // 1343/AC1
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.gate[key] = value
|
||||
b.t.OnAsk(key, 3, value)
|
||||
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
|
||||
if len(asked) != 1 || asked[0].id != ga.Ask.ID {
|
||||
t.Fatalf("published %+v, want one ask under %s with id %s", b.pubs, delivery.TellerModule, ga.Ask.ID)
|
||||
}
|
||||
var got asks.Ask
|
||||
if err := json.Unmarshal(asked[0].body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Digest() != ga.Ask.Digest() || got.Headline != ga.Ask.Headline || got.Explanation != ga.Ask.Explanation || got.ID != delivery.AskID(key) {
|
||||
t.Errorf("the router's ask differs from the gate's: %+v", got)
|
||||
}
|
||||
b.t.OnAsk(key, 3, value) // the revision seen again, as at a restart
|
||||
b.t.OnAsk(key, 4, value) // the key written again with the same ask
|
||||
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 1 {
|
||||
t.Errorf("the ask was published %d times", n)
|
||||
}
|
||||
delete(b.gate, key)
|
||||
b.t.OnAskGone(key)
|
||||
cancels := b.askedOf(asks.CancelSubject(delivery.TellerModule))
|
||||
if len(cancels) != 1 || string(cancels[0].body) != `{"id":"`+ga.Ask.ID+`"}` {
|
||||
t.Fatalf("cancels = %+v", cancels)
|
||||
}
|
||||
b.t.OnAskGone(key)
|
||||
if n := len(b.askedOf(asks.CancelSubject(delivery.TellerModule))); n != 1 {
|
||||
t.Errorf("the ask was cancelled %d times", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarrantThatHoldsWritesOneAnswerAndASecondWritesNone(t *testing.T) { // 1343/AC2
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.gate[key] = value
|
||||
b.t.OnAsk(key, 1, value)
|
||||
w := warrantFor(ga.Ask, "approve")
|
||||
b.deliver(t, w)
|
||||
raw, ok := b.answers.written[delivery.AnswerKey(key)]
|
||||
if !ok || len(b.answers.written) != 1 {
|
||||
t.Fatalf("answers = %v", b.answers.written)
|
||||
}
|
||||
a, err := delivery.DecodeAnswer(raw)
|
||||
if err != nil || a.Option != "approve" || a.Warrant != w.Ask+"@"+w.At.UTC().Format(time.RFC3339Nano) || !strings.Contains(a.Words, "chose Approve") {
|
||||
t.Errorf("answer %+v err %v", a, err)
|
||||
}
|
||||
b.deliver(t, w) // the same warrant again
|
||||
w2 := w
|
||||
w2.At = w.At.Add(time.Minute) // a second warrant for the same ask
|
||||
b.deliver(t, w2)
|
||||
if b.answers.writes != 1 {
|
||||
t.Errorf("the answer was written %d times, want once", b.answers.writes)
|
||||
}
|
||||
if len(b.conds) != 0 {
|
||||
t.Errorf("conditions said: %+v", b.conds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarrantThatDoesNotHoldWritesNoAnswerAndRaisesTheCondition(t *testing.T) { // 1343/AC3
|
||||
other := []delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: headB}, {Repository: "novox/mesh-sdk", Number: 4, Head: headB}}
|
||||
cases := map[string]func(t *testing.T) (*bed, string, asks.Warrant){
|
||||
"heads that differ from the ones the option bound": func(t *testing.T) (*bed, string, asks.Warrant) {
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), other) // the options bind other heads than the ask covers
|
||||
b.gate[key] = value
|
||||
b.t.OnAsk(key, 1, value)
|
||||
return b, key, warrantFor(ga.Ask, "approve")
|
||||
},
|
||||
"an asker that differs": func(t *testing.T) (*bed, string, asks.Warrant) {
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.gate[key] = value
|
||||
b.t.OnAsk(key, 1, value)
|
||||
w := warrantFor(ga.Ask, "approve")
|
||||
w.Asker = "mesh-delivery"
|
||||
return b, key, w
|
||||
},
|
||||
"a digest that differs": func(t *testing.T) (*bed, string, asks.Warrant) {
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.gate[key] = value
|
||||
b.t.OnAsk(key, 1, value)
|
||||
w := warrantFor(ga.Ask, "approve")
|
||||
w.AskDigest = "sha256:0000"
|
||||
return b, key, w
|
||||
},
|
||||
}
|
||||
for name, mk := range cases {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
b, key, w := mk(t)
|
||||
b.deliver(t, w)
|
||||
if len(b.answers.written) != 0 {
|
||||
t.Errorf("an answer was written: %v", b.answers.written)
|
||||
}
|
||||
if len(b.conds) != 1 || b.conds[0].State != asks.StateRaised || b.conds[0].Subject.ID != key ||
|
||||
!strings.HasPrefix(b.conds[0].Explanation, asks.NeedsYou) {
|
||||
t.Fatalf("conditions = %+v", b.conds)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGoodAnswerClearsTheConditionOfAnAskAskedAgain(t *testing.T) { // 1343/AC3 (cleared when a good answer is written)
|
||||
b := newBed(t)
|
||||
other := []delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: headB}, {Repository: "novox/mesh-sdk", Number: 4, Head: headB}}
|
||||
key, bad, gaBad := gateAsk(t, twoHeads(), other)
|
||||
b.gate[key] = bad
|
||||
b.t.OnAsk(key, 1, bad)
|
||||
b.deliver(t, warrantFor(gaBad.Ask, "approve"))
|
||||
if len(b.conds) != 1 || b.conds[0].State != asks.StateRaised {
|
||||
t.Fatalf("conditions = %+v", b.conds)
|
||||
}
|
||||
// The gate writes the ask again, right this time, at a new revision of its key: the second asking, under a new id.
|
||||
_, good, gaGood := gateAskN(t, twoHeads(), twoHeads(), 2)
|
||||
b.gate[key] = good
|
||||
b.t.OnAsk(key, 2, good)
|
||||
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 2 {
|
||||
t.Fatalf("the ask was published %d times, want again at the new revision", n)
|
||||
}
|
||||
b.deliver(t, warrantFor(gaGood.Ask, "approve"))
|
||||
if _, ok := b.answers.written[delivery.AnswerKey(key)]; !ok {
|
||||
t.Fatalf("no answer: %v\n%s", b.answers.written, b.logs.String())
|
||||
}
|
||||
if len(b.conds) != 2 || b.conds[1].State != asks.StateCleared || b.conds[1].Key != b.conds[0].Key {
|
||||
t.Errorf("conditions = %+v", b.conds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAWarrantComingWhileTheTellerWasAwayIsReadFromTheRouter(t *testing.T) {
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.gate[key] = value
|
||||
b.t.OnAsk(key, 1, value)
|
||||
w, _ := json.Marshal(warrantFor(ga.Ask, "approve"))
|
||||
b.records[ga.Ask.ID] = `{"state":"chosen","warrant":` + string(w) + `}`
|
||||
b.t.OnAsk(key, 1, value) // the teller starts again and reads the key at the same revision
|
||||
if _, ok := b.answers.written[delivery.AnswerKey(key)]; !ok {
|
||||
t.Errorf("no answer: %s", b.logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestARewriteWithAskingTwoCancelsAskingOneAndAsksUnderTheNewID(t *testing.T) { // G2 of review round 1
|
||||
b := newBed(t)
|
||||
key, one, g1 := gateAskN(t, twoHeads(), twoHeads(), 1)
|
||||
b.gate[key] = one
|
||||
b.t.OnAsk(key, 1, one)
|
||||
_, two, g2 := gateAskN(t, twoHeads(), twoHeads(), 2)
|
||||
if g1.Ask.ID == g2.Ask.ID {
|
||||
t.Fatal("the two askings share an id")
|
||||
}
|
||||
b.gate[key] = two
|
||||
b.t.OnAsk(key, 2, two)
|
||||
cancels := b.askedOf(asks.CancelSubject(delivery.TellerModule))
|
||||
if len(cancels) != 1 || string(cancels[0].body) != `{"id":"`+g1.Ask.ID+`"}` {
|
||||
t.Fatalf("cancels = %+v, want asking 1's router ask", cancels)
|
||||
}
|
||||
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
|
||||
if len(asked) != 2 || asked[1].id != g2.Ask.ID {
|
||||
t.Fatalf("asks = %+v, want the second under %s", asked, g2.Ask.ID)
|
||||
}
|
||||
if len(b.dropped) != 0 {
|
||||
t.Errorf("the router dropped asks under a reused id: %v", b.dropped)
|
||||
}
|
||||
// The same revision again, and a warrant for the first asking (the gate has moved on), change nothing.
|
||||
b.t.OnAsk(key, 2, two)
|
||||
b.deliver(t, warrantFor(g1.Ask, "approve"))
|
||||
if len(b.answers.written) != 0 || len(b.askedOf(asks.AskSubject(delivery.TellerModule))) != 2 {
|
||||
t.Errorf("answers %v asks %d", b.answers.written, len(b.askedOf(asks.AskSubject(delivery.TellerModule))))
|
||||
}
|
||||
// The warrant of the second names it: its answer's warrant is the router id and the time.
|
||||
w := warrantFor(g2.Ask, "approve")
|
||||
b.deliver(t, w)
|
||||
raw, ok := b.answers.written[delivery.AnswerKey(key)]
|
||||
a, err := delivery.DecodeAnswer(raw)
|
||||
if !ok || err != nil || a.Warrant != g2.Ask.ID+"@"+w.At.UTC().Format(time.RFC3339Nano) {
|
||||
t.Errorf("answer %+v (%v) err %v", a, ok, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAskWhoseWordsChangedUnderTheSameIDIsCancelledAndNotAskedAgain(t *testing.T) {
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.t.OnAsk(key, 1, value)
|
||||
changed := ga
|
||||
changed.Ask.Explanation = "Merges two pull requests, now with a third point.\nReview: approved."
|
||||
v2, err := delivery.EncodeAsk(changed)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.t.OnAsk(key, 2, v2)
|
||||
if n := len(b.askedOf(asks.CancelSubject(delivery.TellerModule))); n != 1 {
|
||||
t.Errorf("cancels = %d", n)
|
||||
}
|
||||
if n := len(b.askedOf(asks.AskSubject(delivery.TellerModule))); n != 1 || len(b.dropped) != 0 {
|
||||
t.Errorf("asks = %d dropped %v: the router would drop a second ask under the id", n, b.dropped)
|
||||
}
|
||||
}
|
||||
|
||||
func TestACapitalisedRepositoryIsSpelledInLowerCaseInTheAct(t *testing.T) { // T1 of review round 1
|
||||
b := newBed(t)
|
||||
heads := []delivery.Head{{Repository: "novox/Mesh-Delivery", Number: 12, Head: headA}}
|
||||
key, err := delivery.MergeAskKey("g", heads)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := delivery.AskID(key)
|
||||
// The act as the gate binds it: its IDOf lowercases the repository (mesh-catalog modules/mesh-delivery-gate model.go).
|
||||
binds, err := asks.ActDigest(asks.Act{"verb": "mesh-delivery.merge", "ask": id, "option": "approve", "heads": "novox/mesh-delivery@" + headA[:12] + "=" + headA})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := asks.Ask{ID: id, Headline: "Merge into main: Mesh-Delivery#12", Explanation: "Merges.", Who: asks.Operator, Expires: now.Add(asks.ApproveLasts - 10*time.Minute),
|
||||
OnExpiry: "the merge stays held", Options: []asks.Option{{ID: "approve", Label: "Approve", Does: "merges", Level: asks.Approve, Binds: binds}}}
|
||||
value, err := delivery.EncodeAsk(delivery.Ask{Group: "g", Heads: heads, Ask: a})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.gate[key] = value
|
||||
b.t.OnAsk(key, 1, value)
|
||||
b.deliver(t, warrantFor(a, "approve"))
|
||||
if _, ok := b.answers.written[delivery.AnswerKey(key)]; !ok {
|
||||
t.Errorf("no answer for the gate's spelling: %s", b.logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAKeyWhoseValueIsNotItsOwnAskIsLeft(t *testing.T) {
|
||||
b := newBed(t)
|
||||
_, value, _ := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.t.OnAsk("merge.g.000000000000", 1, value)
|
||||
b.t.OnAsk("merge.g.000000000000", 2, []byte(`{"v":1`))
|
||||
if len(b.pubs) != 0 {
|
||||
t.Errorf("published %+v", b.pubs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefusedAskIsSaidAsACondition(t *testing.T) { // ADR 0437 §9
|
||||
b := newBed(t)
|
||||
key, value, ga := gateAsk(t, twoHeads(), twoHeads())
|
||||
b.t.OnAsk(key, 1, value)
|
||||
w := asks.Warrant{Ask: ga.Ask.ID, Asker: delivery.TellerModule, Outcome: asks.OutcomeRefused, Words: "no channel can send", At: now}
|
||||
b.deliver(t, w)
|
||||
if len(b.conds) != 1 || b.conds[0].State != asks.StateRaised || !strings.Contains(b.conds[0].Summary, "no channel can send") {
|
||||
t.Errorf("conditions = %+v", b.conds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerToAnEarlierAskingDoesNotBlockTheNextAsking(t *testing.T) { // R2-1 of review round 2
|
||||
b := newBed(t)
|
||||
key, one, g1 := gateAskN(t, twoHeads(), twoHeads(), 1)
|
||||
b.gate[key] = one
|
||||
b.t.OnAsk(key, 1, one)
|
||||
b.deliver(t, warrantFor(g1.Ask, "approve"))
|
||||
first, ok := b.answers.written[delivery.AnswerKey(key)]
|
||||
if a, err := delivery.DecodeAnswer(first); !ok || err != nil || !strings.HasPrefix(a.Warrant, g1.Ask.ID+"@") {
|
||||
t.Fatalf("asking 1's answer: %+v %v %v", a, ok, err)
|
||||
}
|
||||
// The gate rewrites the key at asking 2 (a review record changed). Asking 1's answer stays in the bucket.
|
||||
_, two, g2 := gateAskN(t, twoHeads(), twoHeads(), 2)
|
||||
b.gate[key] = two
|
||||
b.t.OnAsk(key, 2, two)
|
||||
asked := b.askedOf(asks.AskSubject(delivery.TellerModule))
|
||||
if len(asked) != 2 || asked[1].id != g2.Ask.ID {
|
||||
t.Fatalf("asks = %+v, want asking 2 asked under %s", asked, g2.Ask.ID)
|
||||
}
|
||||
w := warrantFor(g2.Ask, "approve")
|
||||
b.deliver(t, w)
|
||||
raw := b.answers.written[delivery.AnswerKey(key)]
|
||||
a, err := delivery.DecodeAnswer(raw)
|
||||
if err != nil || a.Warrant != g2.Ask.ID+"@"+w.At.UTC().Format(time.RFC3339Nano) {
|
||||
t.Fatalf("the key holds %+v (%v), want asking 2's answer", a, err)
|
||||
}
|
||||
// A second warrant for the same asking writes nothing, and a replay of asking 2 asks nothing.
|
||||
w2 := w
|
||||
w2.At = w.At.Add(time.Minute)
|
||||
b.deliver(t, w2)
|
||||
b.t.OnAsk(key, 2, two)
|
||||
b.t.OnAsk(key, 3, two)
|
||||
if b.answers.writes != 2 || len(b.askedOf(asks.AskSubject(delivery.TellerModule))) != 2 || len(b.dropped) != 0 {
|
||||
t.Errorf("writes %d asks %d dropped %v", b.answers.writes, len(b.askedOf(asks.AskSubject(delivery.TellerModule))), b.dropped)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ module mesh-delivery-teller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -76,7 +76,9 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1
|
||||
if m.Module != "mesh-delivery-teller" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "answers", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||
// answers is the teller's answer to each ask; asked is what it asked of the router, which condition it raised and which
|
||||
// gate key and revision each ask was made at (issue 1343).
|
||||
if want := []state{{Name: "answers", History: 5}, {Name: "asked", History: 1}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-delivery-gate.asks"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
@@ -134,7 +136,10 @@ func TestManifestProcess(t *testing.T) { // 1240/AC1
|
||||
}
|
||||
|
||||
func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1240/AC3 (ADR 0437 §8)
|
||||
forbidden := []string{"Subscribe(", "QueueSubscribe", "ChanSubscribe", "time.NewTicker", "time.Tick", "time.NewTimer", "time.AfterFunc", "time.After(", "time.Sleep"}
|
||||
forbidden := []string{"Subscribe(", "QueueSubscribe", "ChanSubscribe", "Respond(", "RespondMsg(", "time.NewTicker", "time.Tick", "time.NewTimer", "time.AfterFunc", "time.After(", "time.Sleep"}
|
||||
// The one subscription the teller makes is to its own `decided` event, the router's word on its asks (ADR 0259, ADR 0437
|
||||
// §6): an event it hears, not a request it answers. It is allowed in decided.go and there only on that subject.
|
||||
const decidedFile = "decided.go"
|
||||
var files int
|
||||
for _, root := range []string{"cmd", "internal"} {
|
||||
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
@@ -150,6 +155,13 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1240/AC3 (ADR
|
||||
return err
|
||||
}
|
||||
for _, f := range forbidden {
|
||||
if f == "Subscribe(" && filepath.Base(path) == decidedFile {
|
||||
if n := strings.Count(string(src), "conn.Subscribe("); n != 1 || !strings.Contains(string(src), "conn.Subscribe(asks.DecidedSubject(module),") ||
|
||||
strings.Count(string(src), f) != n {
|
||||
t.Errorf("%s holds %d subscriptions, and not just one to asks.DecidedSubject(module)", path, strings.Count(string(src), f))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if strings.Contains(string(src), f) {
|
||||
t.Errorf("%s holds %s: the teller serves no request subject and holds no timer", path, f)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
"version": "1",
|
||||
"slug": "teller",
|
||||
"state": [
|
||||
{ "name": "answers", "history": 5 }
|
||||
{ "name": "answers", "history": 5 },
|
||||
{ "name": "asked", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.asks"
|
||||
|
||||
@@ -8,7 +8,9 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-view/internal/view"
|
||||
@@ -109,3 +111,59 @@ func mustHash(t *testing.T, h delivery.Head) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// TestPrintsTheOpenAsksInTheTableAndTheJSON: the command shows what waits for the operator's Approve (1345/AC2). The ask is
|
||||
// the gate's, as mesh-sdk go/delivery EncodeAsk writes it under the key MergeAskKey gives (as internal/view asks_test.go).
|
||||
func TestPrintsTheOpenAsksInTheTableAndTheJSON(t *testing.T) { // 1345/AC2
|
||||
m := view.New()
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: "0123456789abcdef0123456789abcdef01234567"}}
|
||||
key, err := delivery.MergeAskKey("feat/x", heads)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask := asks.Ask{ID: delivery.AskID(key), Headline: "Merge into main: mesh-delivery#12", Explanation: "Merges one pull request.", Who: asks.Operator,
|
||||
Expires: time.Now().Add(time.Hour), OnExpiry: "the merge stays held",
|
||||
Options: []asks.Option{{ID: "approve", Label: "Approve", Does: "merges", Level: asks.Approve, Binds: "sha256:00"}}}
|
||||
value, err := delivery.EncodeAsk(delivery.Ask{Group: "feat/x", Heads: heads, Ask: ask})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Apply(view.Entry{Bucket: delivery.AsksBucket, Key: key, Value: value})
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "c")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
l, err := view.ListenSocket(filepath.Join(dir, "cli.sock"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
go view.ServeSocket(ctx, l, m)
|
||||
t.Cleanup(cancel)
|
||||
socket := filepath.Join(dir, "cli.sock")
|
||||
var out, errOut bytes.Buffer
|
||||
if code := run(nil, socket, &out, &errOut); code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||
}
|
||||
for _, want := range []string{"ASKS", "Merge into main: mesh-delivery#12", "Merges one pull request.", "answer on the phone"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the table lacks %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
out.Reset()
|
||||
if code := run([]string{"--json"}, socket, &out, &errOut); code != 0 {
|
||||
t.Fatalf("exit %d: %s", code, errOut.String())
|
||||
}
|
||||
var got struct {
|
||||
Asks []struct {
|
||||
Key string `json:"key"`
|
||||
Headline string `json:"headline"`
|
||||
How string `json:"how"`
|
||||
} `json:"asks"`
|
||||
}
|
||||
if err := json.Unmarshal(out.Bytes(), &got); err != nil || len(got.Asks) != 1 || got.Asks[0].Key != key ||
|
||||
got.Asks[0].Headline != "Merge into main: mesh-delivery#12" || got.Asks[0].How != "answer on the phone" {
|
||||
t.Errorf("json = %s (%v)", out.String(), err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
// Command mesh-delivery-view-tools serves the read tools of the indexer's modules (novox/hq issue 1562): the runtime launches it for a
|
||||
// call, it reads the indexer's modules bucket through the view's reads (mesh-delivery-indexer.modules) and writes nothing.
|
||||
// The view is the other program: the indexer serves no tool (ADR 0437 §4).
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"mesh-delivery-view/internal/view"
|
||||
)
|
||||
|
||||
// Tools are the read tools over a modules state.
|
||||
func Tools(r view.Reader) []stdio.Tool {
|
||||
return []stdio.Tool{
|
||||
{Name: view.ModuleReadTool,
|
||||
Description: "What the indexer stored for one module under modules.<module>, as stored: its fingerprint (fp, none), the " +
|
||||
"new-form fingerprint (fp2) or why it has none (none2), the inputs it was computed from (inputs2), the module's " +
|
||||
"directory (path) and the toolchains it was computed with. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{
|
||||
"module": map[string]any{"type": "string", "description": "the module's name"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
name, _ := args["module"].(string)
|
||||
return r.Module(name)
|
||||
}},
|
||||
{Name: view.ModulesReadTool,
|
||||
Description: "Every module the indexer holds, each as stored under modules.<module>. With none2 true, only the modules " +
|
||||
"that have no new-form fingerprint, each with the reason (none2). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{
|
||||
"none2": map[string]any{"type": "boolean", "description": "only the modules without a new-form fingerprint"}}},
|
||||
Run: func(args map[string]any) (any, error) {
|
||||
only, _ := args["none2"].(bool)
|
||||
mods, err := r.Modules(only)
|
||||
return map[string]any{"modules": mods}, err
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
state := stdio.State("mesh-delivery-indexer.modules")
|
||||
r := view.Reader{State: view.StateRead{Keys: state.Keys, Get: func(key string) (json.RawMessage, error) {
|
||||
e, err := state.Get(key)
|
||||
if err != nil || e == nil {
|
||||
return nil, err
|
||||
}
|
||||
return e.Value, nil
|
||||
}}}
|
||||
if err := stdio.Serve("", Tools(r)); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-view-tools] %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"mesh-delivery-view/internal/view"
|
||||
)
|
||||
|
||||
type fake map[string]string
|
||||
|
||||
func (f fake) reader() view.Reader {
|
||||
return view.Reader{State: view.StateRead{
|
||||
Keys: func() ([]string, error) {
|
||||
var k []string
|
||||
for key := range f {
|
||||
k = append(k, key)
|
||||
}
|
||||
return k, nil
|
||||
},
|
||||
Get: func(key string) (json.RawMessage, error) {
|
||||
if v, ok := f[key]; ok {
|
||||
return json.RawMessage(v), nil
|
||||
}
|
||||
return nil, nil
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
func run(t *testing.T, r view.Reader, name string, args map[string]any) (any, error) {
|
||||
t.Helper()
|
||||
for _, tl := range Tools(r) {
|
||||
if tl.Name == name {
|
||||
return tl.Run(args)
|
||||
}
|
||||
}
|
||||
t.Fatalf("no tool %s", name)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
const (
|
||||
withFP2 = `{"v":3,"fp":"a1","fp2":"b2","manifest":"m1","inputs2":{"tree":"t1","bases":["x"]},"repository":"novox/r","commit":"c1","path":".","toolchains":{"go":{"version":"go1.27.1","recipe":"2"}}}`
|
||||
withNone = `{"v":3,"none":"no successful trunk build recorded its inputs","none2":"builder's fingerprint not yet known","repository":"novox/r","commit":"c2","path":"modules/b"}`
|
||||
)
|
||||
|
||||
// 1562/AC1: indexer_module answers the key's value as stored: FP, FP2, None2, Inputs2, Path and Toolchains are all there.
|
||||
func TestModuleToolAnswersTheStoredValue(t *testing.T) {
|
||||
r := fake{"a": withFP2, "b": withNone}.reader()
|
||||
for key, want := range map[string]string{"a": withFP2, "b": withNone} {
|
||||
out, err := run(t, r, view.ModuleReadTool, map[string]any{"module": key})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := string(out.(json.RawMessage)); got != want {
|
||||
t.Errorf("%s = %s, want %s", key, got, want)
|
||||
}
|
||||
}
|
||||
out, _ := run(t, r, view.ModuleReadTool, map[string]any{"module": "a"})
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(out.(json.RawMessage), &m); err != nil || m["fp"] != "a1" || m["fp2"] != "b2" || m["path"] != "." || m["inputs2"] == nil || m["toolchains"] == nil {
|
||||
t.Errorf("stored fields are not all there: %v %v", m, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A module with no key, and a name that is no key, are said so.
|
||||
func TestModuleToolRefusesWhatIsNotThere(t *testing.T) {
|
||||
r := fake{"a": withFP2}.reader()
|
||||
if _, err := run(t, r, view.ModuleReadTool, map[string]any{"module": "gone"}); err == nil || !strings.Contains(err.Error(), "no module gone") {
|
||||
t.Errorf("a missing module: %v", err)
|
||||
}
|
||||
if _, err := run(t, r, view.ModuleReadTool, map[string]any{"module": ""}); err == nil {
|
||||
t.Errorf("an empty name was answered")
|
||||
}
|
||||
}
|
||||
|
||||
// 1562/AC1: indexer_modules answers every key as stored, and with none2 only those that say why they have no FP2.
|
||||
func TestModulesToolAnswersEveryKeyAsStored(t *testing.T) {
|
||||
r := fake{"a": withFP2, "b": withNone}.reader()
|
||||
out, err := run(t, r, view.ModulesReadTool, map[string]any{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all := out.(map[string]any)["modules"].(map[string]json.RawMessage)
|
||||
if len(all) != 2 || string(all["a"]) != withFP2 || string(all["b"]) != withNone {
|
||||
t.Errorf("modules = %v", all)
|
||||
}
|
||||
out, err = run(t, r, view.ModulesReadTool, map[string]any{"none2": true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
only := out.(map[string]any)["modules"].(map[string]json.RawMessage)
|
||||
if len(only) != 1 || string(only["b"]) != withNone {
|
||||
t.Errorf("none2 modules = %v", only)
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-view/internal/view"
|
||||
@@ -299,3 +300,93 @@ func TestProcessWholeShowsTheIndexersBucketsOnceTheyExist(t *testing.T) { // 124
|
||||
}
|
||||
t.Fatalf("the snapshot never held the indexer's keys; last: %+v", last)
|
||||
}
|
||||
|
||||
// TestProcessWholeShowsTheOpenAsksAndDropsTheAnsweredOnes starts the program whole before the gate's asks bucket and the
|
||||
// teller's answers bucket exist, then creates them: an ask appears in the snapshot, and leaves it once the teller has an
|
||||
// answer under its key (1345/AC1 against a real bus; the retry is the view's own, every 5 s).
|
||||
func TestProcessWholeShowsTheOpenAsksAndDropsTheAnsweredOnes(t *testing.T) { // 1345/AC1
|
||||
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go srv.Start()
|
||||
if !srv.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
nc, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(nc.Close)
|
||||
js, err := jetstream.New(nc)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||
broker := filepath.Join(dir, "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n", Module: "mesh-delivery-view", User: "mesh-delivery-view", Password: "x"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
socket := filepath.Join(dir, "run", "cli.sock")
|
||||
env := map[string]string{"MESH_BROKER_FILE": broker, "MESH_DELIVERY_SOCKET": socket, "MESH_DELIVERY_WEB": "127.0.0.1:0"}
|
||||
runCtx, stop := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
|
||||
gate, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AsksBucket, History: delivery.AsksHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
teller, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AnswersBucket, History: delivery.AnswersHistory})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: "0123456789abcdef0123456789abcdef01234567"}}
|
||||
key, err := delivery.MergeAskKey("feat/x", heads)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ask := asks.Ask{ID: delivery.AskID(key), Headline: "Merge into main: mesh-delivery#12", Explanation: "Merges one pull request.", Who: asks.Operator,
|
||||
Expires: time.Now().Add(time.Hour), OnExpiry: "the merge stays held",
|
||||
Options: []asks.Option{{ID: "approve", Label: "Approve", Does: "merges", Level: asks.Approve, Binds: "sha256:00"}}}
|
||||
value, err := delivery.EncodeAsk(delivery.Ask{Group: "feat/x", Heads: heads, Ask: ask})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := gate.Put(ctx, key, value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wait := func(check func(view.Snapshot) bool, what string) {
|
||||
t.Helper()
|
||||
var last view.Snapshot
|
||||
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) {
|
||||
if s, err := view.Ask(socket); err == nil {
|
||||
last = s
|
||||
if check(s) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s; last: %+v", what, last)
|
||||
}
|
||||
wait(func(s view.Snapshot) bool {
|
||||
return len(s.Asks) == 1 && s.Asks[0].Key == key && s.Asks[0].How == "answer on the phone"
|
||||
}, "the open ask never showed")
|
||||
answer, err := delivery.EncodeAnswer(delivery.Answer{Option: "approve", Warrant: ask.ID + "@2026-10-18T12:00:00Z", Words: "the operator, via telegram, chose Approve"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := teller.Put(ctx, key, answer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wait(func(s view.Snapshot) bool { return len(s.Asks) == 0 }, "the answered ask never left the list")
|
||||
}
|
||||
|
||||
@@ -136,7 +136,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket} {
|
||||
for _, bucket := range []string{delivery.HaveBucket, view.AssignmentsBucket, view.BuildsBucket, delivery.PresenceBucket, delivery.StuckBucket, delivery.CandidatesBucket, delivery.ModulesBucket, delivery.AsksBucket, delivery.AnswersBucket} {
|
||||
go follow(ctx, js, bucket, model, logger)
|
||||
}
|
||||
select {
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-view
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3 h1:Mu2QiGbL/sZFU5ya4lGUyQOB5OKXPy/KB40XfXTf2EU=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019124204-9ade199740e3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337 h1:cZyFnvDiBpnNx8ifSUG9WA61OHsn7PC2ml2s4eD7AkA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a h1:Vjk2Kq+g/Q86N8CCHq3OatJIJ0JSy6Kh980ekGxNLmE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
package view
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
const askHeadA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
|
||||
// askEntry is a key of the gate's asks as the gate writes it (mesh-sdk go/delivery EncodeAsk, MergeAskKey): the router's ask
|
||||
// whole, under the id its key gives.
|
||||
func askEntry(t *testing.T, group, headline, words string) Entry {
|
||||
t.Helper()
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: askHeadA}}
|
||||
key, err := delivery.MergeAskKey(group, heads)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := asks.Ask{ID: delivery.AskID(key), Headline: headline, Explanation: words, Who: asks.Operator, Expires: t0.Add(23 * time.Hour), OnExpiry: "the merge stays held",
|
||||
Options: []asks.Option{{ID: "approve", Label: "Approve", Does: "merges", Level: asks.Approve, Binds: "sha256:00"}}}
|
||||
v, err := delivery.EncodeAsk(delivery.Ask{Group: group, Heads: heads, Ask: a})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.AsksBucket, Key: key, Value: v, At: t0}
|
||||
}
|
||||
|
||||
// answerEntry is the teller's answer to the first asking of the key; answerToAsking to the asking with this router id. The
|
||||
// answer's warrant is `<router id>@<time>` (teller.go warrantID).
|
||||
func answerEntry(t *testing.T, key string) Entry { return answerToAsking(t, key, delivery.AskID(key)) }
|
||||
|
||||
func answerToAsking(t *testing.T, key, id string) Entry {
|
||||
t.Helper()
|
||||
v, err := delivery.EncodeAnswer(delivery.Answer{Option: "approve", Warrant: id + "@2026-10-18T12:00:00Z", Words: "the operator, via telegram, chose Approve"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return Entry{Bucket: delivery.AnswersBucket, Key: key, Value: v, At: t0}
|
||||
}
|
||||
|
||||
func TestAnAskWithNoAnswerIsListedAndAnAnsweredOneIsNot(t *testing.T) { // 1345/AC1
|
||||
m := New()
|
||||
e := askEntry(t, "feat/x", "Merge into main: mesh-delivery#12", "Merges one pull request.\nReview: approved.")
|
||||
fill(m, e)
|
||||
s := m.Snapshot()
|
||||
if len(s.Asks) != 1 {
|
||||
t.Fatalf("asks = %+v, want the one open ask", s.Asks)
|
||||
}
|
||||
a := s.Asks[0]
|
||||
if a.Key != e.Key || a.Group != "feat/x" || a.Headline != "Merge into main: mesh-delivery#12" || !strings.Contains(a.Words, "Merges one pull request.") ||
|
||||
a.How != "answer on the phone" || len(a.Pulls) != 1 || !strings.Contains(a.Pulls[0], "novox/mesh-delivery#12") || !strings.Contains(a.Pulls[0], askHeadA[:12]) {
|
||||
t.Errorf("ask = %+v", a)
|
||||
}
|
||||
fill(m, answerEntry(t, e.Key))
|
||||
if got := m.Snapshot().Asks; len(got) != 0 {
|
||||
t.Errorf("an answered ask is listed: %+v", got)
|
||||
}
|
||||
// An answer taken away leaves the ask open again; an ask taken away leaves the list.
|
||||
fill(m, Entry{Bucket: delivery.AnswersBucket, Key: e.Key, Deleted: true})
|
||||
if got := m.Snapshot().Asks; len(got) != 1 {
|
||||
t.Errorf("asks after the answer was taken away = %+v", got)
|
||||
}
|
||||
fill(m, Entry{Bucket: delivery.AsksBucket, Key: e.Key, Deleted: true})
|
||||
if got := m.Snapshot().Asks; len(got) != 0 {
|
||||
t.Errorf("a withdrawn ask is listed: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerHeardBeforeItsAskStillHidesTheAsk(t *testing.T) {
|
||||
m := New()
|
||||
e := askEntry(t, "feat/x", "Merge", "Merges.")
|
||||
fill(m, answerEntry(t, e.Key), e)
|
||||
if got := m.Snapshot().Asks; len(got) != 0 {
|
||||
t.Errorf("asks = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreadableAskIsNamedAndNotListed(t *testing.T) { // never dropped silently
|
||||
m := New()
|
||||
fill(m, Entry{Bucket: delivery.AsksBucket, Key: "merge.g.0123456789ab", Value: []byte("{"), At: t0})
|
||||
s := m.Snapshot()
|
||||
if len(s.Asks) != 0 || len(s.Unreadable) != 1 || s.Unreadable[0].Bucket != delivery.AsksBucket {
|
||||
t.Errorf("snapshot = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTableAndPageShowTheAsks(t *testing.T) { // 1345/AC2
|
||||
m := New()
|
||||
fill(m, askEntry(t, "feat/x", "Merge into main: mesh-delivery#12", "Merges one pull request."))
|
||||
var out bytes.Buffer
|
||||
Table(&out, m.Snapshot())
|
||||
for _, want := range []string{"ASKS", "Merge into main: mesh-delivery#12", "Merges one pull request.", "answer on the phone", "novox/mesh-delivery#12"} {
|
||||
if !strings.Contains(out.String(), want) {
|
||||
t.Errorf("the table lacks %q:\n%s", want, out.String())
|
||||
}
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
Web(m).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||
for _, want := range []string{"ASKS", "Merge into main: mesh-delivery#12", "Merges one pull request.", "answer on the phone"} {
|
||||
if !strings.Contains(rec.Body.String(), want) {
|
||||
t.Errorf("the page lacks %q:\n%s", want, rec.Body.String())
|
||||
}
|
||||
}
|
||||
// No ask, no section.
|
||||
out.Reset()
|
||||
Table(&out, New().Snapshot())
|
||||
if strings.Contains(out.String(), "ASKS") {
|
||||
t.Errorf("the table shows an ASKS section with no ask:\n%s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSnapshotsJSONCarriesTheAsks(t *testing.T) { // 1345/AC2
|
||||
m := New()
|
||||
fill(m, askEntry(t, "feat/x", "Merge into main: mesh-delivery#12", "Merges one pull request."))
|
||||
raw, err := json.Marshal(m.Snapshot())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var back struct {
|
||||
Asks []struct {
|
||||
Headline string `json:"headline"`
|
||||
How string `json:"how"`
|
||||
} `json:"asks"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &back); err != nil || len(back.Asks) != 1 || back.Asks[0].How != "answer on the phone" {
|
||||
t.Errorf("json %s: %+v %v", raw, back, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHaveOfARemovedModuleIsShownRemovedNotLeftBehind(t *testing.T) { // 1370: the node-engine writes phase removed
|
||||
m := New()
|
||||
fill(m, haveE(t, "novox", "mesh-delivery", have(t, "b1", delivery.PhaseRemoved, delivery.HealthOK, ""), t0),
|
||||
haveE(t, "novox", "stray", have(t, "b2", delivery.PhaseRunning, delivery.HealthOK, ""), t0))
|
||||
s := m.Snapshot()
|
||||
if r := find(t, s, "novox", "mesh-delivery"); r.Status != StatusRemoved {
|
||||
t.Errorf("row = %+v, want status %q", r, StatusRemoved)
|
||||
}
|
||||
if r := find(t, s, "novox", "stray"); r.Status != StatusLeftBehind {
|
||||
t.Errorf("row = %+v, want left behind", r)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
Table(&out, s)
|
||||
line := ""
|
||||
for _, l := range strings.Split(out.String(), "\n") {
|
||||
if strings.Contains(l, "mesh-delivery") {
|
||||
line = l
|
||||
}
|
||||
}
|
||||
if !strings.Contains(line, "removed") || strings.Contains(line, "left behind") {
|
||||
t.Errorf("table line %q", line)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerToAnEarlierAskingDoesNotHideTheNextAsking(t *testing.T) { // R2-1 of review round 2
|
||||
m := New()
|
||||
e := askEntry(t, "feat/x", "Merge into main: mesh-delivery#12", "Merges one pull request.")
|
||||
fill(m, e, answerEntry(t, e.Key))
|
||||
if got := m.Snapshot().Asks; len(got) != 0 {
|
||||
t.Fatalf("asking 1 is answered and listed: %+v", got)
|
||||
}
|
||||
// The gate rewrites the key at asking 2; only asking 1's answer exists.
|
||||
heads := []delivery.Head{{Repository: "novox/mesh-delivery", Number: 12, Head: askHeadA}}
|
||||
a := asks.Ask{ID: delivery.AskingID(e.Key, 2), Headline: "Merge into main: mesh-delivery#12", Explanation: "Merges one pull request.", Who: asks.Operator,
|
||||
Expires: t0.Add(23 * time.Hour), OnExpiry: "the merge stays held",
|
||||
Options: []asks.Option{{ID: "approve", Label: "Approve", Does: "merges", Level: asks.Approve, Binds: "sha256:00"}}}
|
||||
v, err := delivery.EncodeAsk(delivery.Ask{Group: "feat/x", Heads: heads, Asking: 2, Ask: a})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fill(m, Entry{Bucket: delivery.AsksBucket, Key: e.Key, Value: v, At: t0})
|
||||
if got := m.Snapshot().Asks; len(got) != 1 {
|
||||
t.Fatalf("asking 2 is not listed while only asking 1's answer exists: %+v", got)
|
||||
}
|
||||
fill(m, answerToAsking(t, e.Key, a.ID))
|
||||
if got := m.Snapshot().Asks; len(got) != 0 {
|
||||
t.Errorf("asking 2 is answered and listed: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -53,9 +53,19 @@ func modEntry(t *testing.T, name string, m delivery.Module) Entry {
|
||||
// trunkBuild is a build record as the controller writes it (mesh-sdk go/delivery/testdata/build-record-build-agent.json is
|
||||
// the captured shape: id, state, repository, ref, module, commit, path, source-fingerprint, source-inputs, at).
|
||||
func trunkBuild(module, id, ref, state, fp, at string) []Entry {
|
||||
rec := `{"id":"` + id + `","state":"` + state + `","repository":"novox/mesh-delivery","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/` + module + `","at":"` + at + `"`
|
||||
// an outcome has no state key; a failed one has "failed" (mesh-controller internal/chain/records.go:57-59, :109)
|
||||
rec := `{"id":"` + id + `",`
|
||||
switch state {
|
||||
case "":
|
||||
case "failed":
|
||||
rec += `"failed":"the build failed",`
|
||||
default:
|
||||
rec += `"state":"` + state + `",`
|
||||
}
|
||||
rec += `"repository":"novox/mesh-delivery","ref":"` + ref + `","module":"` + module + `","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/` + module + `","at":"` + at + `"`
|
||||
if fp != "" {
|
||||
rec += `,"source-fingerprint":"` + fp + `"`
|
||||
// the controller writes the inputs whenever it writes the fingerprint (mesh-sdk testdata/build-record-build-agent.json)
|
||||
rec += `,"source-fingerprint":"` + fp + `","source-inputs":{"Module":"` + module + `","Tree":"t","Bases":null,"Contexts":null,"Toolchains":null,"Unpinned":""}`
|
||||
}
|
||||
rec += `}`
|
||||
return []Entry{
|
||||
@@ -103,8 +113,8 @@ func TestSnapshotListsTheCandidateGroups(t *testing.T) { // 1241/AC1
|
||||
|
||||
func TestFingerprintAgainstTheNewestTrunkBuild(t *testing.T) { // 1241/AC2
|
||||
m := New()
|
||||
fill(m, trunkBuild("a", "build-1", "main", "built", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m, trunkBuild("d", "build-2", "main", "built", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m, trunkBuild("a", "build-1", "main", "", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m, trunkBuild("d", "build-2", "main", "", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m,
|
||||
modEntry(t, "a", delivery.Module{FP: fF, Repository: "novox/mesh-delivery", Commit: hA}),
|
||||
modEntry(t, "d", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA}),
|
||||
@@ -127,10 +137,10 @@ func TestFingerprintAgainstTheNewestTrunkBuild(t *testing.T) { // 1241/AC2
|
||||
|
||||
func TestOnlyTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1241/AC2 (the newest key is only a hint)
|
||||
m := New()
|
||||
fill(m, trunkBuild("a", "build-1", "main", "built", fG, "2026-10-19T09:00:00Z")...)
|
||||
fill(m, trunkBuild("a", "build-2", "main", "built", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m, trunkBuild("a", "build-3", "main", "failed", fG, "2026-10-19T11:00:00Z")...) // newer, not built
|
||||
fill(m, trunkBuild("a", "build-4", "feat/x", "built", fG, "2026-10-19T12:00:00Z")...) // newer, not trunk; also the newest key
|
||||
fill(m, trunkBuild("a", "build-1", "main", "", fG, "2026-10-19T09:00:00Z")...)
|
||||
fill(m, trunkBuild("a", "build-2", "main", "", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m, trunkBuild("a", "build-3", "main", "failed", fG, "2026-10-19T11:00:00Z")...) // newer, not built
|
||||
fill(m, trunkBuild("a", "build-4", "feat/x", "", fG, "2026-10-19T12:00:00Z")...) // newer, not trunk; also the newest key
|
||||
fill(m, modEntry(t, "a", delivery.Module{FP: fF, Repository: "novox/mesh-delivery", Commit: hA}))
|
||||
if r := fpRow(t, m.Snapshot(), "a"); r.State != FPSame || r.Build != fF {
|
||||
t.Errorf("a = %+v, want same with the 10:00 trunk build's fingerprint", r)
|
||||
@@ -140,7 +150,7 @@ func TestOnlyTheNewestSuccessfulTrunkBuildCounts(t *testing.T) { // 1241/AC2 (th
|
||||
func TestBuildFingerprintIsReadWithTheSDKRecord(t *testing.T) { // 1241/AC3
|
||||
// The record as the controller's encoder wrote it (mesh-sdk go/delivery/testdata/build-record-build-agent.json), whole,
|
||||
// with its source-inputs under their Go field names: the view reads source-fingerprint from it.
|
||||
const rec = `{"id":"build-1792395447088936520","state":"built","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","manifest-digest":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","source-inputs":{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""},"made":[{"name":"server","kind":"image","reference":"registry.example/build-agent@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"}],"at":"2026-10-19T09:38:00Z"}`
|
||||
const rec = `{"id":"build-1792395447088936520","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","manifest-digest":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","source-inputs":{"Module":"build-agent","Tree":"11c0b4f90cae0d6e4ef9cc48cbfdf9f6dbe52bca","Bases":["sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"],"Contexts":{"server":"narrow:d51ca744fd11af59d616c858f76c27b3e58e16e11fbdc0c720eb9d7d19d55fac"},"Toolchains":null,"Unpinned":""},"made":[{"name":"server","kind":"image","reference":"registry.example/build-agent@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"}],"at":"2026-10-19T09:38:00Z"}`
|
||||
const fp = "src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7"
|
||||
m := New()
|
||||
fill(m, Entry{Bucket: BuildsBucket, Key: delivery.BuildKey("build-agent", "build-1792395447088936520"), Value: []byte(rec), At: t0},
|
||||
@@ -153,7 +163,7 @@ func TestBuildFingerprintIsReadWithTheSDKRecord(t *testing.T) { // 1241/AC3
|
||||
|
||||
func shown(t *testing.T) *Model {
|
||||
m := New()
|
||||
fill(m, trunkBuild("a", "build-1", "main", "built", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m, trunkBuild("a", "build-1", "main", "", fF, "2026-10-19T10:00:00Z")...)
|
||||
fill(m,
|
||||
candidate(t, "feat/x", pullOf("novox/mesh-delivery", 12, hA, "a"), pullOf("novox/mesh-catalog", 300, hB, "c")),
|
||||
modEntry(t, "a", delivery.Module{FP: fG, Repository: "novox/mesh-delivery", Commit: hA}),
|
||||
@@ -188,3 +198,23 @@ func TestUnreadableIndexerValuesAreNamed(t *testing.T) { // never dropped silent
|
||||
t.Errorf("snapshot = %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlanAskedBuildWithNoRefIsATrunkBuild(t *testing.T) { // 1281/AC5
|
||||
// The build record of module mesh-delivery from mesh-catalog, build-1792435539606203626, as `mesh-controller.command
|
||||
// builds` showed it on 2026-10-19 at about 21:20 local: built on ace at 20:45 local, the repository the clone URL, the
|
||||
// source fingerprint below, and no ref (a delivery plan asked it after a merge; the listing shows no "at main").
|
||||
// No reading command shows the whole record: its path is the module's directory in the catalogue, and its
|
||||
// source-inputs are mailu's live ones (mesh-sdk go/delivery/fingerprint_test.go) with the module's name put in.
|
||||
const live = "src1:f79e2eedc2689cf6b3591ebfdb81ec65899131122fb2fe3dd9e384091d8a2a6d"
|
||||
const rec = `{"id":"build-1792435539606203626","repository":"http://novox.internal:20000/novox/mesh-catalog.git",` +
|
||||
`"module":"mesh-delivery","commit":"9efdd87a","path":"modules/mesh-delivery","at":"2026-10-19T18:45:00Z","source-fingerprint":"` + live + `",` +
|
||||
`"source-inputs":{"Module":"mesh-delivery","Tree":"cfc6fa8ff5fc1e220132e51ade653b9d3f387b3e","Bases":["sha256:25f3cfeaceca14921366af4d1240b56457ef46273bdb508c7b0e8f469f6fd228"],"Contexts":null,"Toolchains":null,"Unpinned":""}}`
|
||||
m := New()
|
||||
fill(m,
|
||||
Entry{Bucket: BuildsBucket, Key: "mesh-delivery.build.build-1792435539606203626", Value: []byte(rec), At: t0},
|
||||
Entry{Bucket: BuildsBucket, Key: "mesh-delivery.newest", Value: []byte("build-1792435539606203626"), At: t0},
|
||||
modEntry(t, "mesh-delivery", delivery.Module{FP: live, Repository: "novox/mesh-catalog", Commit: hA}))
|
||||
if r := fpRow(t, m.Snapshot(), "mesh-delivery"); r.State != FPSame || r.Build != live {
|
||||
t.Errorf("mesh-delivery = %+v, want same as the no-ref build's fingerprint", r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package view
|
||||
|
||||
// The read tools of the indexer's modules (novox/hq issue 1562): what the indexer stored under modules.<m>, answered as
|
||||
// stored. The new-form fingerprint (FP2, None2, Inputs2), the reason it has none, and the path and toolchains it was computed
|
||||
// with are only there. The view reads the bucket through its reads; the reads write nothing.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The read tools, named as the module is: indexer_ and the verb.
|
||||
const (
|
||||
ModuleReadTool = "indexer_module"
|
||||
ModulesReadTool = "indexer_modules"
|
||||
)
|
||||
|
||||
// StateRead reads the indexer's modules state, as the view's reads grant it: its keys and a key's stored value (nil for a key with none).
|
||||
type StateRead struct {
|
||||
Keys func() ([]string, error)
|
||||
Get func(key string) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
// Reader answers the read tools from a modules state.
|
||||
type Reader struct{ State StateRead }
|
||||
|
||||
// Module is the stored value of modules.<module>, exactly as the indexer wrote it.
|
||||
func (r Reader) Module(module string) (json.RawMessage, error) {
|
||||
module = strings.TrimSpace(module)
|
||||
key, err := delivery.ModulesKey(module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.State.Get == nil {
|
||||
return nil, fmt.Errorf("the modules state cannot be read here")
|
||||
}
|
||||
raw, err := r.State.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("no module %s in the modules state", module)
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// Modules is every key of the modules state with its stored value; with none2 only the modules that have no new-form
|
||||
// fingerprint and say why (a None2 reason), which is what the fingerprint's check reads live.
|
||||
func (r Reader) Modules(none2 bool) (map[string]json.RawMessage, error) {
|
||||
if r.State.Keys == nil || r.State.Get == nil {
|
||||
return nil, fmt.Errorf("the modules state cannot be read here")
|
||||
}
|
||||
keys, err := r.State.Keys()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := map[string]json.RawMessage{}
|
||||
for _, k := range keys {
|
||||
raw, err := r.State.Get(k)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
if none2 {
|
||||
var m struct {
|
||||
None2 string `json:"none2"`
|
||||
}
|
||||
if json.Unmarshal(raw, &m) != nil || m.None2 == "" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out[k] = raw
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -111,9 +111,9 @@ func rowText(r Row) []string {
|
||||
if r.Error != "" {
|
||||
what = "unreadable: " + r.Error
|
||||
}
|
||||
if r.Status == StatusLeftBehind && what == "" {
|
||||
if (r.Status == StatusLeftBehind || r.Status == StatusRemoved) && what == "" {
|
||||
what = r.Status
|
||||
} else if r.Status == StatusLeftBehind {
|
||||
} else if r.Status == StatusLeftBehind || r.Status == StatusRemoved {
|
||||
what = r.Status + "; " + what
|
||||
}
|
||||
return []string{r.Node, r.Module, r.Assigned, r.Have, r.Phase, r.Health, since(r.Since), r.Detail, what}
|
||||
@@ -152,6 +152,15 @@ func fingerprintText(s Snapshot) [][]string {
|
||||
return out
|
||||
}
|
||||
|
||||
// askText is one line per open ask: its key, headline, words on one line, the pull requests it covers and how to answer.
|
||||
func askText(s Snapshot) [][]string {
|
||||
var out [][]string
|
||||
for _, a := range s.Asks {
|
||||
out = append(out, []string{a.Key, a.Headline, strings.Join(strings.Fields(strings.ReplaceAll(a.Words, "\n", " / ")), " "), strings.Join(a.Pulls, ", "), a.How})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var head = []string{"NODE", "MODULE", "ASSIGNED", "HAVE", "PHASE", "HEALTH", "SINCE", "DETAIL", "WAITS"}
|
||||
|
||||
// Table prints the snapshot as a table.
|
||||
@@ -184,6 +193,13 @@ func Table(w io.Writer, s Snapshot) {
|
||||
row(cells)
|
||||
}
|
||||
}
|
||||
if a := askText(s); len(a) > 0 {
|
||||
fmt.Fprintln(tw)
|
||||
row([]string{"ASKS", "HEADLINE", "WORDS", "PULL REQUESTS", "ANSWER"})
|
||||
for _, cells := range a {
|
||||
row(cells)
|
||||
}
|
||||
}
|
||||
if f := fingerprintText(s); len(f) > 0 {
|
||||
fmt.Fprintln(tw)
|
||||
row([]string{"MODULE", "FINGERPRINT", "INDEXER AGAINST BUILD"})
|
||||
@@ -227,6 +243,7 @@ var page = template.Must(template.New("p").Parse(`<!doctype html><html><head><me
|
||||
<body><h1>What each node runs</h1><table><tr>{{range .Head}}<th>{{.}}</th>{{end}}</tr>
|
||||
{{range .Rows}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>
|
||||
{{if .Groups}}<h2>CANDIDATES</h2><table><tr><th>GROUP</th><th>PULL REQUEST</th><th>HEAD</th><th>MODULES</th></tr>{{range .Groups}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Asks}}<h2>ASKS</h2><table><tr><th>ASK</th><th>HEADLINE</th><th>WORDS</th><th>PULL REQUESTS</th><th>ANSWER</th></tr>{{range .Asks}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Fingerprints}}<h2>FINGERPRINTS</h2><table><tr><th>MODULE</th><th>STATE</th><th>INDEXER AGAINST BUILD</th></tr>{{range .Fingerprints}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Offline}}<h2>OFFLINE</h2><table>{{range .Offline}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
{{if .Stuck}}<h2>STUCK</h2><table><tr><th>BUCKET</th><th>KEY</th><th>WAITED FOR</th><th>SINCE</th></tr>{{range .Stuck}}<tr>{{range .}}<td>{{.}}</td>{{end}}</tr>{{end}}</table>{{end}}
|
||||
@@ -260,6 +277,6 @@ func Web(m *Model) http.Handler {
|
||||
for _, u := range snap.Unreadable {
|
||||
bad = append(bad, []string{u.Bucket, u.Key, u.Error})
|
||||
}
|
||||
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Fingerprints": fingerprintText(snap), "Offline": off, "Stuck": stuck, "Unreadable": bad})
|
||||
page.Execute(w, map[string]any{"Head": head, "Rows": rows, "Groups": groupText(snap), "Asks": askText(snap), "Fingerprints": fingerprintText(snap), "Offline": off, "Stuck": stuck, "Unreadable": bad})
|
||||
})
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user