novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'The bridge carries each module's manifest and lock hashes, and the indexer writes manifests by digest and each module's new-form inputs (design 47 R4c-2, issues 1448, 1449, 1450, 1444, 1451, 1458)' (#10) from feat/r4c2-new-keys into main

Browse Source
This commit was merged in pull request #10.
This commit is contained in:
mesh-tools
2026-10-20 19:31:05 +00:00
parent 2520cc4a88 c9f742d5f2
commit 340bf64388
30 changed files with 1791 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go
+32
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"regexp"
"strings"
"time"
@@ -89,3 +90,34 @@ func treesVia(ctx context.Context, call caller) bridge.Trees {
return a.Root, a.Trees, nil
}
}
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
const getFileTool = "gitea.gitea_get_file"
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
// recognises it as /: 404 /).
var notFound = regexp.MustCompile(`: 404 `)
// filesVia asks the forge for a file through call. A file the forge says is not there is bridge.ErrNotFound.
func filesVia(ctx context.Context, call caller) bridge.Files {
return func(owner, repo, path, ref string) ([]byte, error) {
raw, err := call(ctx, getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
if err != nil {
if notFound.MatchString(err.Error()) {
return nil, fmt.Errorf("%w: %v", bridge.ErrNotFound, err)
}
return nil, err
}
var a struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
}
return []byte(*a.File.Content), nil
}
}
modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go
+76 -7
View File
@@ -4,6 +4,8 @@ package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -29,8 +31,8 @@ import (
// bucket performed is the bridge's own (its state), and the five forge tools of the gate's acts are invoked (issue 1341, 1342).
//
// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls and
// gitea.gitea_commit_trees, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls,
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
// WirePattern). The consumer's durable is <node>_<module> on EVENTS (nats.go consumerDurable). The controller's own
// user keeps all rights, as it has.
const busConf = `
@@ -59,6 +61,10 @@ accounts {
"mesh.mod.gitea.call.gitea_open_pulls.n1~mesh-delivery-forge-bridge"
"mesh.mod.gitea.tool.gitea_commit_trees"
"mesh.mod.gitea.tool.gitea_commit_trees.*"
"mesh.mod.gitea.call.gitea_get_file.*.n1~mesh-delivery-forge-bridge"
"mesh.mod.gitea.call.gitea_get_file.n1~mesh-delivery-forge-bridge"
"mesh.mod.gitea.tool.gitea_get_file"
"mesh.mod.gitea.tool.gitea_get_file.*"
"mesh.mod.gitea.tool.gitea_open_pulls"
"mesh.mod.gitea.tool.gitea_open_pulls.*"
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_performed.>"
@@ -160,6 +166,21 @@ type bed struct {
ctx context.Context
done <-chan error
reply func(string)
// files sets what the stand-in forge answers to gitea_get_file from then on; it is asked with the tool's arguments.
files func(func(owner, repo, path, ref string) (string, bool))
}
// getFileReply is the reply envelope for gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile
// answer it ({file: {path, ref, sha, size, content}}), and for a path the forge does not have the error client.ts request
// writes: `Gitea API <path>: 404 <body>`.
func getFileReply(path, ref, content string, found bool) []byte {
if !found {
b, _ := json.Marshal(map[string]any{"error": "Gitea API /repos/novox/mesh-delivery/contents/" + path + "?ref=" + ref + `: 404 {"message":"GetContentsOrList","url":"https://forge.example/api/swagger"}`})
return b
}
b, _ := json.Marshal(map[string]any{"result": map[string]any{"file": map[string]any{"path": path, "ref": ref, "sha": "0123456789abcdef0123456789abcdef01234567",
"size": len(content), "content": content}}, "node": "n0"})
return b
}
// startBed is startBridge with a step between the streams and the consumer: prepare writes what the streams hold before
@@ -222,6 +243,26 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
t.Fatal(err)
}
// A stand-in answers the forge's gitea_get_file: by default a module.json of {} at every path and no other file.
var fmu sync.Mutex
filesOf := func(_, _, path, _ string) (string, bool) { return "{}", strings.HasSuffix(path, "module.json") }
getFile, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_get_file", func(m *nats.Msg) {
var a struct{ Owner, Repo, Path, Ref string }
_ = json.Unmarshal(m.Data, &a)
fmu.Lock()
f := filesOf
fmu.Unlock()
content, found := f(a.Owner, a.Repo, a.Path, a.Ref)
_ = m.Respond(getFileReply(a.Path, a.Ref, content, found))
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = getFile.Unsubscribe() })
if err := ctl.Flush(); err != nil {
t.Fatal(err)
}
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-forge-bridge", User: "n1.mesh-delivery-forge-bridge", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
@@ -232,7 +273,8 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
t.Cleanup(stop)
return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() }}
return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() },
files: func(f func(owner, repo, path, ref string) (string, bool)) { fmu.Lock(); filesOf = f; fmu.Unlock() }}
}
// TestBridgeWritesPullMessages starts the program the way the node does (its entry, run, with MESH_BROKER_FILE).
@@ -270,7 +312,7 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true}
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
@@ -295,8 +337,31 @@ const merge = "89abcdef0123456789abcdef0123456789abcdef"
// TestBridgeWritesMergedMessage has a stand-in answer the forge's gitea_commit_trees on its tool subject
// (mesh-control internal/broker/nats.go invokedSubjects: mesh.mod.<module>.tool.<tool>), in the reply envelope of
// mesh-sdk go/process/call.go ({result, error, node}).
func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4
ctl, js, ctx, _ := startBridge(t, `{"result":{"pulls":[]}}`)
func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
bed := startBed(t, `{"result":{"pulls":[]}}`, nil)
ctl, js, ctx := bed.ctl, bed.js, bed.ctx
// The stand-in forge's files are two real files of this repository (this module's module.json and go.sum, read from the
// working tree), in gitea_get_file's real answer shape: the root has a manifest only, modules/a a manifest and go.sum.
manifest, err := os.ReadFile("../../module.json")
if err != nil {
t.Fatal(err)
}
sum, err := os.ReadFile("../../go.sum")
if err != nil {
t.Fatal(err)
}
bed.files(func(_, _, path, ref string) (string, bool) {
if ref != merge {
t.Errorf("the forge was asked for %s at %s, want the merge commit %s", path, ref, merge)
}
switch path {
case "module.json", "modules/a/module.json":
return string(manifest), true
case "modules/a/go.sum":
return string(sum), true
}
return "", false
})
var asked chan []byte = make(chan []byte, 4)
sub, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) {
asked <- m.Data
@@ -342,7 +407,9 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"}}
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
@@ -637,3 +704,5 @@ func forgeMerged(t *testing.T, b *bed) []string {
}
return out
}
func sha256Sum(b []byte) []byte { s := sha256.Sum256(b); return s[:] }
modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go
+3 -3
View File
@@ -115,8 +115,8 @@ type publisher interface {
// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
// delivers it again (the server's ack wait is the only clock).
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, logger *log.Logger) {
msg, err := bridge.Convert(m.Subject(), m.Data(), trees)
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) {
msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files)
switch {
case errors.Is(err, bridge.ErrRetry):
logger.Printf("%v; heard again in %s", err, retryDelay)
@@ -238,7 +238,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// Bounded: the turn is held, and no sync or other event goes on without it.
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
defer cancel()
handle(hctx, m, rec, treesVia(hctx, call), logger)
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger)
})
},
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go
+66 -7
View File
@@ -6,7 +6,9 @@ import (
"encoding/json"
"errors"
"log"
"slices"
"strings"
"sync"
"testing"
"time"
@@ -14,6 +16,8 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-forge-bridge/internal/bridge"
)
const head = "0123456789abcdef0123456789abcdef01234567"
@@ -64,7 +68,7 @@ func body(m map[string]any) []byte {
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
t.Fatalf("published on %v", pub.subjects)
}
@@ -80,7 +84,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{}
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
if len(pub.subjects) != 0 {
t.Errorf("a message was written: %v", pub.subjects)
}
@@ -95,7 +99,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
var logs bytes.Buffer
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
if m.acked != 0 || m.termd != 0 {
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
}
@@ -117,7 +121,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/
call := func(context.Context, string, any) (json.RawMessage, error) {
return nil, errors.New("the tree listing of novox/mesh-delivery was cut")
}
handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0))
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
if len(pub.subjects) != 0 {
t.Errorf("a message was written: %v", pub.subjects)
}
@@ -136,13 +140,19 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{}
var keys []string
var args []any
var mu sync.Mutex // the files are asked for concurrently (issue 1458)
call := func(_ context.Context, key string, body any) (json.RawMessage, error) {
mu.Lock()
keys, args = append(keys, key), append(args, body)
mu.Unlock()
if key == "gitea.gitea_get_file" { // the manifests, 1448; the trees are what this test is about
return json.RawMessage(`{"file":{"path":"p","sha":"0123","size":2,"content":"{}"}}`), nil
}
return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil
}
handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0))
if len(keys) != 1 || keys[0] != "gitea.gitea_commit_trees" {
t.Fatalf("calls %v", keys)
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 {
t.Fatalf("calls %v, want the trees once and first", keys)
}
if got, _ := json.Marshal(args[0]); string(got) != `{"owner":"novox","repo":"mesh-delivery","sha":"89abcdef0123456789abcdef0123456789abcdef"}` {
t.Errorf("arguments %s", got)
@@ -171,3 +181,52 @@ func TestAnAnswerWithoutARootIsNotUsed(t *testing.T) { // 1236 (a shape the tool
t.Error("an answer that is not an object was taken")
}
}
// The answer of gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile write it, and the
// error of a path the forge does not have as client.ts request writes it (`Gitea API ${path}: ${status} ${text}`).
const getFileAnswer = `{"file":{"path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","sha":"0123","size":4,"content":"x y\n"}}`
const getFile404 = "Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=89abcdef: 404 {\"message\":\"GetContentsOrList\",\"url\":\"https://forge.example/api/swagger\"}"
func TestFilesViaAsksByOwnerRepoPathRefAndTakesTheContent(t *testing.T) { // 1448/AC4
var keys []string
var args []any
call := func(_ context.Context, key string, body any) (json.RawMessage, error) {
keys, args = append(keys, key), append(args, body)
return json.RawMessage(getFileAnswer), nil
}
got, err := filesVia(context.Background(), call)("novox", "mesh-delivery", "modules/a/go.sum", "89abcdef0123456789abcdef0123456789abcdef")
if err != nil || string(got) != "x y\n" {
t.Fatalf("got %q err %v", got, err)
}
if len(keys) != 1 || keys[0] != "gitea.gitea_get_file" {
t.Fatalf("calls %v", keys)
}
if b, _ := json.Marshal(args[0]); string(b) != `{"owner":"novox","path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","repo":"mesh-delivery"}` {
t.Errorf("arguments %s", b)
}
}
func TestFilesViaTellsNotFoundFromOtherErrorsAndOtherShapes(t *testing.T) { // 1448/AC3
call := func(err error, raw string) caller {
return func(context.Context, string, any) (json.RawMessage, error) { return json.RawMessage(raw), err }
}
if _, err := filesVia(context.Background(), call(errors.New(getFile404), ""))("o", "r", "p", "s"); !errors.Is(err, bridge.ErrNotFound) {
t.Errorf("a 404 is %v, want ErrNotFound", err)
}
if _, err := filesVia(context.Background(), call(errors.New("Gitea API /repos/o/r/contents/p: 502 bad gateway"), ""))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) {
t.Errorf("a 502 is %v, want an error that is not ErrNotFound", err)
}
for _, raw := range []string{`{"file":null}`, `{}`, `[1]`, `{"file":{"path":"p"}}`} {
if _, err := filesVia(context.Background(), call(nil, raw))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) {
t.Errorf("the answer %s gave %v, want an error", raw, err)
}
}
}
// someFiles is a forge that has a module.json at every path and no other file, for the tests that are not about the files.
func someFiles(_, _, path, _ string) ([]byte, error) {
if strings.HasSuffix(path, "module.json") {
return []byte("{}"), nil
}
return nil, bridge.ErrNotFound
}
modules/mesh-delivery-forge-bridge/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-forge-bridge/go.sum
+2 -6
View File
@@ -1,9 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9 h1:XL1dbtSleGAixg6FX4gEnSG/AJxRG/tWw6/7U1RuzHc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go
+121 -4
View File
@@ -2,13 +2,17 @@
package bridge
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"path"
"regexp"
"slices"
"strconv"
"strings"
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -34,6 +38,17 @@ const Trunk = "main"
// (the root directory spelled ""), as the forge's gitea_commit_trees answers (issue 1234/AC4).
type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error)
// ErrNotFound says the forge has no file at that path and ref (the forge's "404", issue 1448).
var ErrNotFound = errors.New("the forge has no such file")
// Files asks the forge for one file's bytes at a ref (a commit), as the forge's gitea_get_file answers; a file the forge
// says is not there is ErrNotFound, any other failure any other error (issue 1448).
type Files func(owner, repo, path, ref string) ([]byte, error)
// LockFiles are the lock files read beside a module's manifest, by name relative to its directory (novox/hq ADR 0465
// clause 1: go.sum for a Go module, package-lock.json for a Node one).
var LockFiles = []string{"go.sum", "package-lock.json"}
// Message is a message for FORGE: the event name (forge.<owner>.<name>.<kind>) and its JSON body.
type Message struct {
Event string
@@ -72,7 +87,7 @@ func unreadable(subject, why string, args ...any) error {
// is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
func Convert(subject string, data []byte, trees Trees) (Message, error) {
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
var action string
switch subject {
case UpdatedSubject:
@@ -99,13 +114,17 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) {
return Message{}, unreadable(subject, "no branch")
}
if subject == MergedSubject && p.Base == Trunk {
return merged(subject, p, trees)
return merged(subject, p, trees, files)
}
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Action: action}
// The directories are the forge's word only when it said it could read them (module_dirs_said).
if action == delivery.PullUpdated && p.ModuleDirsSaid {
f.Dirs, f.DirsKnown = p.ModuleDirs, true
var err error
if f.Manifests, f.Locks, err = inputsOf(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
}
}
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
if err != nil {
@@ -118,8 +137,97 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) {
return Message{Event: event, Data: body}, nil
}
// MaxReadsInFlight is how many files of the forge inputsOf asks for at once. A merge into a repository of many
// modules (131 in the catalogue) reads three files for each, one after another inside the bound of a handled event
// (30 s) that leaves about 76 ms a call; eight at once keeps the forge's load small and the read well inside the bound
// (issue 1458).
const MaxReadsInFlight = 8
// read is one file asked for: the path in the directory dir, and whether it is the module's manifest.
type read struct {
dir, path string
manifest bool
raw []byte
err error
}
// inputsOf reads, at ref, the module.json of each directory (the root spelled "") and the lock files beside it
// (LockFiles): the manifests as the forge answered them and the lock files' hex sha256 by name. A manifest the forge
// does not have is an error, since the directory was said to hold one; a lock file it does not have is left out; any other
// failure is an error (issue 1448).
//
// The files are asked for with at most MaxReadsInFlight calls at once (issue 1458). The result does not depend on the
// order the calls finish in, and the error is that of the first failing file in the order of dirs and LockFiles. After
// a failure no further file is asked for, and inputsOf returns only when every call it started has returned.
func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) {
at := func(dir, name string) string {
if dir == "" {
return name
}
return dir + "/" + name
}
var reads []*read
for _, dir := range dirs {
reads = append(reads, &read{dir: dir, path: at(dir, "module.json"), manifest: true})
for _, name := range LockFiles {
reads = append(reads, &read{dir: dir, path: at(dir, name)})
}
}
var (
wg sync.WaitGroup
mu sync.Mutex
failed bool
slots = make(chan struct{}, MaxReadsInFlight)
)
for _, r := range reads {
slots <- struct{}{}
mu.Lock()
stop := failed
mu.Unlock()
if stop {
<-slots
break
}
wg.Add(1)
go func(r *read) {
defer func() { <-slots; wg.Done() }()
r.raw, r.err = files(owner, repo, r.path, ref)
if r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)) {
mu.Lock()
failed = true
mu.Unlock()
}
}(r)
}
wg.Wait()
manifests = map[string]string{}
locks = map[string]map[string]string{}
for _, r := range reads {
switch {
case r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)):
return nil, nil, fmt.Errorf("%s: %w", r.path, r.err)
case r.err != nil: // a lock file the forge does not have
case r.manifest:
manifests[r.dir] = string(r.raw)
default:
sum := sha256.Sum256(r.raw)
if locks[r.dir] == nil {
locks[r.dir] = map[string]string{}
}
locks[r.dir][path.Base(r.path)] = hex.EncodeToString(sum[:])
}
}
if len(manifests) == 0 {
manifests = nil
}
if len(locks) == 0 {
locks = nil
}
return manifests, locks, nil
}
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
func merged(subject string, p pull, trees Trees) (Message, error) {
func merged(subject string, p pull, trees Trees, files Files) (Message, error) {
if !sha40.MatchString(p.MergeSHA) {
return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
}
@@ -131,12 +239,21 @@ func merged(subject string, p pull, trees Trees) (Message, error) {
if len(dirs) == 0 {
return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s", ErrNoModules, subject, repository, p.MergeSHA)
}
names := make([]string, 0, len(dirs))
for dir := range dirs {
names = append(names, dir)
}
slices.Sort(names)
manifests, locks, err := inputsOf(p.Owner, p.Repo, p.MergeSHA, names, files)
if err != nil {
return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
}
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs})
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks})
if err != nil {
return Message{}, unreadable(subject, "%v", err)
}
modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go
+14 -14
View File
@@ -47,7 +47,7 @@ const updatedSubject = "mesh.mod.gitea.event.pull.updated"
const closedSubject = "mesh.mod.gitea.event.pull.closed"
func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; the bus is the integration test)
msg, err := Convert(updatedSubject, updated(nil), nil)
msg, err := Convert(updatedSubject, updated(nil), nil, anyFiles)
if err != nil {
t.Fatal(err)
}
@@ -59,14 +59,14 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion;
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true}
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
}
func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
msg, err := Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil)
msg, err := Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, anyFiles)
if err != nil {
t.Fatal(err)
}
@@ -75,7 +75,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown)
}
// Said and empty is known and empty, not unknown.
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil)
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil, anyFiles)
if err != nil {
t.Fatal(err)
}
@@ -84,7 +84,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown)
}
// Dirs listed without the forge saying it could read them are not trusted.
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil)
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil, anyFiles)
if err != nil {
t.Fatal(err)
}
@@ -94,7 +94,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
}
func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
msg, err := Convert(closedSubject, closed(), nil)
msg, err := Convert(closedSubject, closed(), nil, anyFiles)
if err != nil {
t.Fatal(err)
}
@@ -130,7 +130,7 @@ func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5
"closed no head": {closedSubject, mustDrop(closed(), "head_sha"), "head"},
"closed no repo": {closedSubject, mustDrop(closed(), "repo"), "repo"},
} {
msg, err := Convert(c.subject, c.data, nil)
msg, err := Convert(c.subject, c.data, nil, anyFiles)
if err == nil {
t.Errorf("%s: a message %q was made", name, msg.Event)
continue
@@ -157,7 +157,7 @@ func mustDrop(body []byte, key string) []byte {
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235
for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} {
if _, err := Convert(s, updated(nil), nil); !errors.Is(err, ErrNotHandled) {
if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) {
t.Errorf("%s: err = %v, want ErrNotHandled", s, err)
}
}
@@ -200,7 +200,7 @@ func treesOf(calls *[]call, root string, trees map[string]string, err error) Tre
func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
var calls []call
answer := map[string]string{"modules/a": "t1", "modules/b": "t2"}
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil))
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil), anyFiles)
if err != nil {
t.Fatal(err)
}
@@ -212,7 +212,7 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
t.Fatal(err)
}
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
Head: head, Merge: merge, Root: "r0", Trees: answer}
Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}}
if !delivery.SameForge(f, want) {
t.Errorf("message = %+v, want %+v", f, want)
}
@@ -223,7 +223,7 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
var calls []call
msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil))
msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
if err != nil {
t.Fatal(err)
}
@@ -243,7 +243,7 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the conversion; the delay is in main_test)
var calls []call
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")))
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")), anyFiles)
if !errors.Is(err, ErrRetry) || msg.Data != nil {
t.Fatalf("err = %v data = %s, want ErrRetry and no message", err, msg.Data)
}
@@ -256,7 +256,7 @@ func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the co
func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (notes: a repository with no module.json)
var calls []call
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil))
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil), anyFiles)
if !errors.Is(err, ErrNoModules) || msg.Data != nil {
t.Errorf("err = %v data = %s", err, msg.Data)
}
@@ -264,7 +264,7 @@ func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (note
func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236
var calls []call
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil))
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
t.Errorf("err = %v calls = %v", err, calls)
}
modules/mesh-delivery-forge-bridge/internal/bridge/files_test.go
+281
View File
@@ -0,0 +1,281 @@
package bridge
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"reflect"
"sort"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the
// module.json and the go.sum of modules/mesh-delivery-forge-bridge (two directories up). The forge's own answer shape
// is mesh-catalog modules/gitea tools/index.ts gitea_get_file; the bridge sees only the bytes (issue 1448).
func real(t *testing.T, name string) []byte {
t.Helper()
b, err := os.ReadFile("../../" + name)
if err != nil {
t.Fatal(err)
}
return b
}
func hexSum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
type fileCall struct{ owner, repo, path, ref string }
// forgeFiles answers as the forge does for a path: its bytes, ErrNotFound for a path in none, the error in errs.
type forgeFiles struct {
have map[string][]byte
errs map[string]error
mu sync.Mutex
calls []fileCall
}
func (f *forgeFiles) get(owner, repo, path, ref string) ([]byte, error) {
f.mu.Lock()
f.calls = append(f.calls, fileCall{owner, repo, path, ref})
f.mu.Unlock()
if err, ok := f.errs[path]; ok {
return nil, err
}
if b, ok := f.have[path]; ok {
return b, nil
}
return nil, ErrNotFound
}
// anyFiles is a forge that has a module.json at every path and no lock file, for tests that are not about the files.
func anyFiles(owner, repo, path, ref string) ([]byte, error) {
if strings.HasSuffix(path, "module.json") {
return []byte("{}"), nil
}
return nil, ErrNotFound
}
func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
manifest, sum := real(t, "module.json"), real(t, "go.sum")
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}}
var calls []call
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get)
if err != nil {
t.Fatal(err)
}
f, err := delivery.DecodeForge(msg.Data)
if err != nil {
t.Fatal(err)
}
if want := map[string]string{"modules/a": string(manifest), "modules/b": `{"module":"b"}`}; !reflect.DeepEqual(f.Manifests, want) {
t.Errorf("manifests = %v", f.Manifests)
}
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
t.Errorf("locks = %v, want %v", f.Locks, want)
}
}
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
manifest := real(t, "module.json")
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest}}
msg, err := Convert(updatedSubject, updated(nil), nil, forge.get)
if err != nil {
t.Fatal(err)
}
f, _ := delivery.DecodeForge(msg.Data)
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 0 {
t.Errorf("manifests %v locks %v", f.Manifests, f.Locks)
}
if len(forge.calls) == 0 || forge.calls[0].ref != head {
t.Errorf("calls %v, want the head %s", forge.calls, head)
}
// Dirs not known: neither field, and the forge is not asked.
forge = &forgeFiles{}
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, forge.get)
if err != nil {
t.Fatal(err)
}
f, _ = delivery.DecodeForge(msg.Data)
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
}
}
func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3
boom := errors.New("Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=x: 502 bad gateway")
trees := func() Trees { var c []call; return treesOf(&c, "r0", map[string]string{"modules/a": "t1"}, nil) }
for name, forge := range map[string]*forgeFiles{
"a manifest error": {errs: map[string]error{"modules/a/module.json": boom}},
"a manifest not found": {},
"a lock file error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/go.sum": boom}},
"a second lock error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/package-lock.json": boom}},
} {
msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
if !errors.Is(err, ErrRetry) || msg.Data != nil {
t.Errorf("%s: err = %v data = %s, want ErrRetry and no message", name, err, msg.Data)
}
_, err = Convert(updatedSubject, updated(nil), nil, forge.get)
if !errors.Is(err, ErrRetry) {
t.Errorf("%s (pull): err = %v, want ErrRetry", name, err)
}
}
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
if err != nil {
t.Fatal(err)
}
if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil {
t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks)
}
}
func TestTheFilesAreAskedAtTheMergeCommitByOwnerRepoPathAndRef(t *testing.T) { // 1448/AC4 (the call's arguments; the manifest is in manifest_test)
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
var calls []call
if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); err != nil {
t.Fatal(err)
}
want := []fileCall{{"novox", "mesh-delivery", "modules/a/module.json", merge}, {"novox", "mesh-delivery", "modules/a/go.sum", merge},
{"novox", "mesh-delivery", "modules/a/package-lock.json", merge}}
sort.Slice(forge.calls, func(i, j int) bool { return forge.calls[i].path < forge.calls[j].path }) // they are asked for concurrently
sort.Slice(want, func(i, j int) bool { return want[i].path < want[j].path })
if !reflect.DeepEqual(forge.calls, want) {
t.Errorf("calls = %v, want %v", forge.calls, want)
}
}
// slowForge answers like a forge that takes d a call: a module.json of the directory's name everywhere, a go.sum in
// the directories whose number is even and in the root, and counts the calls in flight.
type slowForge struct {
d time.Duration
mu sync.Mutex
inFlight, most, started, n int
failOn string
finished int
}
func (f *slowForge) get(_, _, path, _ string) ([]byte, error) {
f.mu.Lock()
f.inFlight++
f.started++
if f.inFlight > f.most {
f.most = f.inFlight
}
f.mu.Unlock()
time.Sleep(f.d)
defer func() { f.mu.Lock(); f.inFlight--; f.finished++; f.mu.Unlock() }()
switch {
case path == f.failOn:
return nil, errors.New("Gitea API " + path + ": 502 bad gateway")
case strings.HasSuffix(path, "module.json"):
return []byte(`{"module":"` + path + `"}`), nil
case strings.HasSuffix(path, "go.sum") && (path == "go.sum" || path[len(path)-8]%2 == 0):
return []byte("sum of " + path), nil
}
return nil, ErrNotFound
}
func manyDirs(n int) []string {
dirs := []string{""}
for i := 1; i < n; i++ {
dirs = append(dirs, fmt.Sprintf("modules/m%02d", i))
}
return dirs
}
func TestFortyDirectoriesAreReadEightAtATimeWithTheResultOfOneAtATime(t *testing.T) { // 1458/AC1
dirs := manyDirs(40)
slow := &slowForge{d: 10 * time.Millisecond}
manifests, locks, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
if err != nil {
t.Fatal(err)
}
if slow.most > MaxReadsInFlight || slow.most < 2 {
t.Errorf("at most %d calls were in flight, want more than one and at most %d", slow.most, MaxReadsInFlight)
}
if slow.started != 40*3 {
t.Errorf("%d calls, want 120", slow.started)
}
// The same read one file at a time.
wantM, wantL := map[string]string{}, map[string]map[string]string{}
one := &slowForge{}
for _, dir := range dirs {
m, l, err := inputsOfOne(dir, one.get)
if err != nil {
t.Fatal(err)
}
wantM[dir] = m
if l != nil {
wantL[dir] = l
}
}
if !reflect.DeepEqual(manifests, wantM) || !reflect.DeepEqual(locks, wantL) {
t.Errorf("the concurrent read differs from a one-at-a-time read: %d manifests %d locks, want %d and %d", len(manifests), len(locks), len(wantM), len(wantL))
}
if len(locks) == 0 || len(locks) == 40 {
t.Errorf("%d directories with locks, the fixture should have some and not all", len(locks))
}
}
// inputsOfOne reads one directory with a single call at a time, as the bridge did before issue 1458.
func inputsOfOne(dir string, files Files) (string, map[string]string, error) {
at := func(name string) string {
if dir == "" {
return name
}
return dir + "/" + name
}
raw, err := files("o", "r", at("module.json"), "x")
if err != nil {
return "", nil, err
}
var locks map[string]string
for _, name := range LockFiles {
b, err := files("o", "r", at(name), "x")
if errors.Is(err, ErrNotFound) {
continue
}
if err != nil {
return "", nil, err
}
if locks == nil {
locks = map[string]string{}
}
s := sha256.Sum256(b)
locks[name] = hex.EncodeToString(s[:])
}
return string(raw), locks, nil
}
func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testing.T) { // 1458/AC2
dirs := manyDirs(40)
slow := &slowForge{d: 10 * time.Millisecond, failOn: "modules/m05/go.sum"}
_, _, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
if err == nil || !strings.Contains(err.Error(), "modules/m05/go.sum") || !strings.Contains(err.Error(), "502") {
t.Fatalf("err = %v, want the failing file named", err)
}
slow.mu.Lock()
defer slow.mu.Unlock()
if slow.inFlight != 0 || slow.finished != slow.started {
t.Errorf("%d calls in flight, %d of %d finished, when the read returned", slow.inFlight, slow.finished, slow.started)
}
if slow.started >= 40*3 {
t.Errorf("%d calls were made, want the read to stop asking after the failure", slow.started)
}
// Through Convert it is ErrRetry.
var calls []call
dirsMap := map[string]string{}
for _, d := range dirs[1:] {
dirsMap[d] = "t"
}
slow2 := &slowForge{failOn: "modules/m05/module.json"}
if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", dirsMap, nil), slow2.get); !errors.Is(err, ErrRetry) {
t.Errorf("err = %v, want ErrRetry", err)
}
}
modules/mesh-delivery-forge-bridge/manifest_test.go
+1 -1
View File
@@ -93,7 +93,7 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view",
"gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
"gitea.gitea_get_file", "gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
modules/mesh-delivery-forge-bridge/module.json
+2 -1
View File
@@ -25,7 +25,8 @@
"gitea.gitea_note_append",
"gitea.gitea_add_comment",
"gitea.gitea_open_pulls",
"gitea.gitea_commit_trees"
"gitea.gitea_commit_trees",
"gitea.gitea_get_file"
],
"own-secrets": {
"broker": "${dir:state}/broker"
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go
+62
View File
@@ -145,6 +145,10 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) (
if err != nil {
t.Fatal(err)
}
// The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449).
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket, History: delivery.ManifestsHistory}); err != nil {
t.Fatal(err)
}
builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuildsBucket, History: 5})
if err != nil {
t.Fatal(err)
@@ -203,3 +207,61 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9
t.Errorf("module = %+v within 5 s, want fp %s", m, want)
}
}
// TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the
// hash of its go.sum (the bridge's Manifests and Locks, issue 1448) makes, within 5 s, modules.avahi with its manifest digest,
// its inputs and the reason it has no fingerprint of the new form, and manifests.<digest> with the manifest byte for byte.
func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4
// The manifest is avahi's module.json from mesh-catalog at a67f01963f0d9ef0913a63720b89e573ca7a16ef (copied into the
// index package's testdata); the lock hash is the sha256 of modules/avahi/go.sum there.
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil {
t.Fatal(err)
}
if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil {
t.Fatal(err)
}
})
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
Trees: map[string]string{"modules/avahi": "newtree"},
Manifests: map[string]string{"modules/avahi": string(manifest)},
Locks: map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil {
t.Fatal(err)
}
var m delivery.Module
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := modules.Get(ctx, "avahi"); err == nil {
if m, err = delivery.DecodeModule(e.Value()); err == nil {
break
}
}
}
digest := delivery.ManifestDigest(manifest)
if m.Manifest != digest || m.None2 != "toolchain not yet reported" || m.FP2 != "" || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" ||
m.Inputs2.Locks["tools"] != goSum {
t.Errorf("module = %+v within 5 s, want its manifest digest %s, its inputs and none2 'toolchain not yet reported'", m, digest)
}
kv, err := js.KeyValue(ctx, delivery.ManifestsBucket)
if err != nil {
t.Fatal(err)
}
e, err := kv.Get(ctx, digest)
if err != nil {
t.Fatalf("manifests.%s: %v", digest, err)
}
if got, err := delivery.DecodeManifest(e.Value()); err != nil || got.Module != "avahi" || got.JSON != string(manifest) {
t.Errorf("manifest = %+v err %v, want avahi and the file's bytes", got, err)
}
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go
+5 -1
View File
@@ -251,6 +251,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
manifests, err := bucket(delivery.ManifestsBucket)
if err != nil {
return err
}
builds, err := bucket(delivery.BuildsBucket)
if err != nil {
return err
@@ -270,7 +274,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
errs := make(chan error, 1)
go func() { errs <- follow(ctx, w, each) }()
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Builds: known, Logf: logger.Printf}
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf}
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
modules/mesh-delivery-indexer/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-indexer/go.sum
+2 -8
View File
@@ -1,11 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337 h1:cZyFnvDiBpnNx8ifSUG9WA61OHsn7PC2ml2s4eD7AkA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a h1:Vjk2Kq+g/Q86N8CCHq3OatJIJ0JSy6Kh980ekGxNLmE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-indexer/internal/index/candidate_manifests_test.go
+105
View File
@@ -0,0 +1,105 @@
package index
import (
"context"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func manifestPull(t *testing.T, number int, head, manifest string) delivery.Forge {
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
if manifest != "" {
f.Manifests = map[string]string{"modules/avahi": manifest}
}
return f
}
func candidateOf(t *testing.T, st *memStore) delivery.Candidate {
t.Helper()
key, err := delivery.CandidatesKey("feat/x")
if err != nil {
t.Fatal(err)
}
raw, ok := st.vals[key]
if !ok {
t.Fatalf("candidates holds %d keys, none %s", len(st.vals), key)
}
c, err := delivery.DecodeCandidate(raw)
if err != nil {
t.Fatal(err)
}
return c
}
func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t *testing.T) { // 1451/AC1
avahi := catalogueManifest(t, "avahi")
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
t.Fatal(err)
}
c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check
want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"}
if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || c.Pulls[0].Fps["avahi"] != want {
t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want)
}
if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) {
t.Errorf("fpset = %q, want %q", c.FPSet, delivery.FingerprintSet(c.Pulls))
}
// The same pull request heard again without its directories (the head unchanged) keeps what it had.
again := manifestPull(t, 301, h1, "")
again.Dirs, again.DirsKnown = nil, false
if err := ix.Apply(context.Background(), again); err != nil {
t.Fatal(err)
}
if c2 := candidateOf(t, ix.Store.(*memStore)); c2.Pulls[0].Fps["avahi"] != want || c2.FPSet != c.FPSet {
t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet)
}
}
func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/AC2
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, "")); err != nil {
t.Fatal(err)
}
c := candidateOf(t, ix.Store.(*memStore))
if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.FPSet != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" {
t.Errorf("candidate = %+v, want the module avahi and no Fps and no fpset", c)
}
}
func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3
avahi := catalogueManifest(t, "avahi")
// The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory.
other := catalogueManifest(t, "ca-trust")
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
t.Fatal(err)
}
before := candidateOf(t, ix.Store.(*memStore))
writes := ix.Store.(*memStore).writes
err := ix.Apply(context.Background(), manifestPull(t, 302, h2, other))
if !errors.Is(err, ErrUnreadable) {
t.Fatalf("err = %v, want a refusal", err)
}
for _, w := range []string{catalogueRepo + "#301", catalogueRepo + "#302", `"avahi"`} {
if !strings.Contains(err.Error(), w) {
t.Errorf("the refusal %q does not name %s", err, w)
}
}
// Candidate.check refuses the group too, which is what the refusal stands for.
if _, encErr := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat.x", HeadsHash: before.HeadsHash, Pulls: []delivery.CandidatePull{before.Pulls[0], {
Head: delivery.Head{Repository: catalogueRepo, Number: 302, Head: h2}, Modules: []string{"avahi"},
Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(other)), None2: NoneNoPreview}}}}}); encErr == nil {
t.Error("the candidate check accepts two manifests for one module")
}
if ix.Store.(*memStore).writes != writes {
t.Error("a partial write was made")
}
if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.FPSet != before.FPSet {
t.Errorf("the candidate changed: %+v", after)
}
}
modules/mesh-delivery-indexer/internal/index/candidates.go
+60 -7
View File
@@ -43,8 +43,10 @@ type Indexer struct {
Store Store // candidates
// Modules is the modules bucket; nil writes no fingerprints.
Modules Store
Builds *Builds
Logf func(format string, args ...any)
// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
Manifests Store
Builds *Builds
Logf func(format string, args ...any)
mu sync.Mutex // guards source, which the source watch and the forge messages both reach
source sourceNow
@@ -72,6 +74,13 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
return err
}
}
var digests map[string]string
if f.Kind == delivery.ForgePull {
var err error
if digests, err = ix.keepManifests(ctx, f); err != nil {
return err
}
}
if trunkLike[f.Branch] {
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
return nil
@@ -89,7 +98,12 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
}
cur = &c
}
next := ix.next(cur, f)
next := ix.next(cur, f, digests)
if next != nil {
if err := ix.refuseDisagreement(*next); err != nil {
return nil, false, err
}
}
switch {
case next == nil && cur == nil, next != nil && cur != nil && delivery.SameCandidate(*cur, *next):
return nil, false, nil
@@ -185,7 +199,12 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte,
}
// next is the group after the message; nil when the group is left with no pull request.
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Candidate {
//
// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of
// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
// directory.
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate {
var pulls []delivery.CandidatePull
var base uint64
if cur != nil {
@@ -202,13 +221,20 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can
switch {
case f.DirsKnown:
for _, dir := range f.Dirs {
if m := ix.Builds.ModuleOf(f.Repository, dir); !slices.Contains(entry.Modules, m) {
m := ix.Builds.ModuleOf(f.Repository, dir)
if !slices.Contains(entry.Modules, m) {
entry.Modules = append(entry.Modules, m)
}
if digest, ok := digests[dir]; ok {
if entry.Fps == nil {
entry.Fps = map[string]delivery.CandidateModule{}
}
entry.Fps[m] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview}
}
}
case at >= 0 && pulls[at].Head.Head == f.Head:
// Its directories are not known now, for the head the group already holds: what it knew stands.
entry.Modules = pulls[at].Modules
entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps
}
if at >= 0 {
pulls[at] = entry
@@ -230,5 +256,32 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can
heads[i] = p.Head
}
hash, _ := delivery.HeadsHash(heads)
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls}
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
}
// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445).
const NoneNoPreview = "no merge preview is computed"
// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the
// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is
// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull
// request is judged again.
func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error {
first := map[string]delivery.CandidatePull{}
for _, p := range c.Pulls {
for module, m := range p.Fps {
before, ok := first[module]
if !ok {
first[module] = p
continue
}
if before.Fps[module] != m {
err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written",
ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest)
return err
}
}
}
return nil
}
modules/mesh-delivery-indexer/internal/index/inputs.go
+115
View File
@@ -0,0 +1,115 @@
package index
import (
"fmt"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The reasons a module has no fingerprint of the new form while the indexer writes none for any module (novox/hq
// ADR 0465 clause 2; issue 1444). Toolchains missing is delivery.ToolchainNoReport.
const (
// None2NoManifest is for a module whose directory the merged message carries no manifest of (an older bridge).
None2NoManifest = "the message carries no manifest"
// None2Builder is for a module with no artifact of a toolchain: only the builder's fingerprint is missing.
None2Builder = "builder's fingerprint not yet known"
// None2Context is for a module an artifact of which reads another repository's tree (today's form gives it none
// either, NoneContext).
None2Context = "context not pinned"
)
// lockFile is the lock file of a language's artifact that resolves packages when it is built, by name in the module's
// directory (the bridge reads these, internal/bridge LockFiles; issue 1450).
var lockFile = map[string]string{"go": "go.sum", "typescript": "package-lock.json"}
// moduleInputs is what a module's manifest and lock hashes give the indexer at a merge (issue 1450).
type moduleInputs struct {
// Kept is the new-form inputs the merge gave the module; nil when the manifest could not be read or gave none.
Kept *delivery.KeptInputs2
// Languages is the language of each artifact that has a toolchain (a bundle or a package of go or typescript), by
// artifact name. The toolchain versions are joined to them when the build modules report them (issue 1444, the
// watching half); the SDK's KeptInputs2 keeps no languages, so they are read from the manifest again then.
Languages map[string]string
// None2 is why the module has no fingerprint of the new form.
None2 string
}
// baseFP answers a base module's new-form fingerprint, or why it has none.
type baseFP func(module string) (fp, none string)
// inputsOf turns the manifest of a module (raw, as the forge answered it), the tree of its directory and the lock
// hashes of that directory (lock file name to hash) into its new-form inputs. The manifest is read by the catalogue's
// parser; a manifest it refuses, or one that is of another module, leaves Kept nil and says why.
//
// The rules are the controller's builder (mesh-controller internal/builder/builder.go): a bundle or a package of a
// language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages,
// resolves packages when it is built (wantsPackages, :1017), so it has an entry in Locks, with the hash of its
// language's lock file or an empty hash when none was carried (the fingerprint is then none, "no lock"); an archive,
// an image and an upstream resolve nothing and have no toolchain. A base is a build.on entry that names a module.
func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs {
m, err := catalogue.ParseManifest([]byte(raw))
if err != nil {
return moduleInputs{None2: "the manifest cannot be read: " + err.Error()}
}
if m.Module != module {
return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)}
}
if tree == "" {
return moduleInputs{None2: "the merge names no tree for the module's directory"}
}
kept := delivery.KeptInputs2{Tree: tree}
langs := map[string]string{}
contexts := false
var missing []string
if m.Build != nil {
for _, a := range m.Build.Artifacts {
if a.Context != nil {
contexts = true
}
if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactPackage {
continue
}
name, ok := lockFile[a.Language]
if !ok {
continue
}
langs[a.Name] = a.Language
if kept.Locks == nil {
kept.Locks = map[string]string{}
}
kept.Locks[a.Name] = locks[name]
}
seen := map[string]bool{}
for _, on := range m.Build.On {
if on.Module == "" || seen[on.Module] {
continue
}
seen[on.Module] = true
if fp, none := base(on.Module); fp != "" {
kept.Bases = append(kept.Bases, fp)
} else {
missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", on.Module, none))
}
}
}
sort.Strings(kept.Bases)
sort.Strings(missing)
out := moduleInputs{Kept: &kept}
if len(langs) > 0 {
out.Languages = langs
}
switch {
case contexts:
out.None2 = None2Context
case len(missing) > 0:
out.None2 = strings.Join(missing, "; ")
case len(langs) > 0:
out.None2 = delivery.ToolchainNoReport
default:
out.None2 = None2Builder
}
return out
}
modules/mesh-delivery-indexer/internal/index/inputs_test.go
+146
View File
@@ -0,0 +1,146 @@
package index
import (
"context"
"reflect"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// aBase is the new-form fingerprint of a base module, the one SourceFingerprint2 gives for the inputs of avahi at a tree.
func aBase(t *testing.T) string {
t.Helper()
fp, none := delivery.SourceFingerprint2(delivery.SourceInputs2{Module: "mesh-tools", Tree: "basetree"})
if none != "" {
t.Fatal(none)
}
return fp
}
func noBases(string) (string, string) { return "", "none are held in this test" }
func TestAGoModuleKeepsItsTreeAndTheHashOfItsLockFile(t *testing.T) { // 1450/AC1
in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, noBases)
want := &delivery.KeptInputs2{Tree: "avahitree", Locks: map[string]string{"tools": avahiGoSum}}
if !reflect.DeepEqual(in.Kept, want) {
t.Errorf("kept = %+v, want %+v", in.Kept, want)
}
if !reflect.DeepEqual(in.Languages, map[string]string{"tools": "go"}) {
t.Errorf("languages = %v, want tools: go", in.Languages)
}
// Through the indexer: the merge's tree of the module's directory and the lock hashes of that directory, kept in modules.avahi.
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "avahitree", "modules/other": "x"}),
map[string]string{"modules/avahi": catalogueManifest(t, "avahi")}, map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "avahi")
if !ok || !reflect.DeepEqual(m.Inputs2, want) || m.Manifest != delivery.ManifestDigest([]byte(catalogueManifest(t, "avahi"))) {
t.Errorf("module = %+v (%v), want inputs %+v and the manifest digest", m, ok, want)
}
}
func TestAnArtifactThatResolvesPackagesAndHasNoLockFileHasAnEmptyHash(t *testing.T) { // 1450/AC2
in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", nil, noBases)
if in.Kept == nil || !reflect.DeepEqual(in.Kept.Locks, map[string]string{"tools": ""}) {
t.Fatalf("kept = %+v, want Locks {tools: \"\"}", in.Kept)
}
// The fingerprint of such inputs is none, "no lock" (mesh-sdk SourceFingerprint2).
if fp, none := delivery.SourceFingerprint2(in.Kept.With("avahi", nil)); fp != "" || none != "no lock" {
t.Errorf("fingerprint = %q none %q, want none 'no lock'", fp, none)
}
// A TypeScript bundle's lock file is package-lock.json, not go.sum.
ts := inputsOf("audit-logger", catalogueManifest(t, "audit-logger"), "t", map[string]string{"go.sum": "wrong", "package-lock.json": avahiGoSum}, noBases)
if ts.Kept == nil || !reflect.DeepEqual(ts.Kept.Locks, map[string]string{"code": avahiGoSum}) || !reflect.DeepEqual(ts.Languages, map[string]string{"code": "typescript"}) {
t.Errorf("typescript: kept %+v languages %v", ts.Kept, ts.Languages)
}
// netcheck has a Go and a TypeScript bundle: each takes its own language's lock file.
both := inputsOf("netcheck", catalogueManifest(t, "netcheck"), "t", map[string]string{"go.sum": "g", "package-lock.json": "p"}, noBases)
if both.Kept == nil || !reflect.DeepEqual(both.Kept.Locks, map[string]string{"tools-go": "g", "tools-typescript": "p"}) {
t.Errorf("netcheck: kept %+v", both.Kept)
}
}
// aModuleWithABase is avahi's manifest (mesh-catalog a67f0196) with the one build.on entry that names a module as the
// controller's own test has it (mesh-controller internal/builder/standing_on_test.go:22): no manifest of the sources names
// a module as a base, only vendors' images.
func aModuleWithABase(t *testing.T) string {
t.Helper()
raw := catalogueManifest(t, "avahi")
with := strings.Replace(raw, `"build": {`, `"build": {
"on": [{"arg": "RUNTIME_BASE", "module": "mesh-tools", "artifact": "runtime"}],`, 1)
if with == raw {
t.Fatal("avahi's manifest has no build")
}
return with
}
func TestABaseEntersByTheFingerprintOfItsModuleOrNamesItsAbsence(t *testing.T) { // 1450/AC3
fp := aBase(t)
has := func(module string) (string, string) {
if module != "mesh-tools" {
t.Errorf("asked for the base %q", module)
}
return fp, ""
}
in := inputsOf("avahi", aModuleWithABase(t), "t", map[string]string{"go.sum": avahiGoSum}, has)
if in.Kept == nil || !reflect.DeepEqual(in.Kept.Bases, []string{fp}) {
t.Errorf("bases = %+v, want [%s]", in.Kept, fp)
}
none := inputsOf("avahi", aModuleWithABase(t), "t", map[string]string{"go.sum": avahiGoSum}, noBases)
if none.Kept == nil || len(none.Kept.Bases) != 0 || !strings.Contains(none.None2, "mesh-tools") || !strings.Contains(none.None2, "base") {
t.Errorf("kept %+v none2 %q, want no bases and none2 naming the base mesh-tools", none.Kept, none.None2)
}
// Through the indexer: modules.mesh-tools holds a fingerprint of the new form (a record written by a later release).
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
mods.put("mesh-tools", mustEncode(t, delivery.Module{Repository: catalogueRepo, Commit: merge, None: "x", FP2: fp, Manifest: strings.Repeat("a", 64)}))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t"}), map[string]string{"modules/avahi": aModuleWithABase(t)}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if m, _ := moduleOf(t, mods, "avahi"); m.Inputs2 == nil || !reflect.DeepEqual(m.Inputs2.Bases, []string{fp}) {
t.Errorf("module = %+v, want the base's fingerprint", m)
}
}
func mustEncode(t *testing.T, m delivery.Module) []byte {
t.Helper()
b, err := delivery.EncodeModule(m)
if err != nil {
t.Fatal(err)
}
return b
}
func TestAnArtifactOfNoToolchainHasNoLanguageAndNoLock(t *testing.T) { // 1450/AC4
in := inputsOf("ca-trust", catalogueManifest(t, "ca-trust"), "t", map[string]string{"go.sum": avahiGoSum}, noBases)
if in.Kept == nil || len(in.Kept.Locks) != 0 || len(in.Languages) != 0 {
t.Errorf("kept %+v languages %v, want neither locks nor languages for an archive", in.Kept, in.Languages)
}
}
func TestAManifestTheParserRefusesLeavesNoInputsAndSaysWhy(t *testing.T) { // 1450/AC5
for name, raw := range map[string]string{
"not json": "not json",
"unknown field": strings.Replace(catalogueManifest(t, "avahi"), `"version": "1",`, `"version": "1", "scope": "everything",`, 1),
} {
in := inputsOf("avahi", raw, "t", nil, noBases)
if in.Kept != nil || !strings.HasPrefix(in.None2, "the manifest cannot be read: ") || len(in.None2) <= len("the manifest cannot be read: ") {
t.Errorf("%s: kept %+v none2 %q, want nil and 'the manifest cannot be read: <error>'", name, in.Kept, in.None2)
}
}
if in := inputsOf("someone-else", catalogueManifest(t, "avahi"), "t", nil, noBases); in.Kept != nil || !strings.Contains(in.None2, "avahi") {
t.Errorf("a manifest of another module: kept %+v none2 %q", in.Kept, in.None2)
}
}
func TestAContextIsNotPinned(t *testing.T) { // 1450 (the rule 1444 keeps: context not pinned, as today's form)
// build-agent's manifest from mesh-controller at f2d7bc94 has an image artifact built from a context in another repository.
in := inputsOf("build-agent", fixture(t, "controller-f2d7bc94-build-agent"), "t", nil, noBases)
if in.Kept == nil || in.None2 != None2Context {
t.Errorf("kept %+v none2 %q, want none2 %q", in.Kept, in.None2, None2Context)
}
}
modules/mesh-delivery-indexer/internal/index/manifests.go
+83
View File
@@ -0,0 +1,83 @@
package index
import (
"context"
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests
// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the
// module a build record names the directory the path of.
func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string {
var m struct {
Module string `json:"module"`
}
if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" {
return m.Module
}
name, _ := ix.Builds.Module(repository, dir)
return name
}
// keepManifest writes manifests.<digest> for a manifest as the forge answered it, create-once (issue 1449): a key that
// exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for
// one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the
// digest. A manifest that cannot be a value (empty, not UTF-8, not JSON) is an error the caller says; it is never tried
// again. With no manifests store nothing is written and the digest is still answered.
func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string, error) {
digest := delivery.ManifestDigest([]byte(raw))
if ix.Manifests == nil {
return digest, nil
}
key, err := delivery.ManifestsKey(digest)
if err != nil {
return "", err
}
data, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
if err != nil {
return "", fmt.Errorf("%w: manifests.%s: %v", errCannotBeAValue, key, err)
}
err = ix.Manifests.Create(ctx, key, data)
if err == nil {
return digest, nil
}
if !errors.Is(err, ErrConflict) {
return "", fmt.Errorf("writing manifests.%s: %w", key, err)
}
held, _, found, err := ix.Manifests.Get(ctx, key)
if err != nil {
return "", fmt.Errorf("reading manifests.%s: %w", key, err)
}
if !found {
return "", fmt.Errorf("manifests.%s was there and is not: written again at the next delivery", key)
}
if cur, err := delivery.DecodeManifest(held); err == nil && delivery.SameManifest(cur, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw}) {
return digest, nil
}
return "", fmt.Errorf("manifests.%s holds another value than the manifest of %s, which hashes to it", key, module)
}
// keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory.
// A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest.
func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) {
digests := make(map[string]string, len(f.Manifests))
for dir, raw := range f.Manifests {
module := ix.moduleNameOf(f.Repository, dir, raw)
digest, err := ix.keepManifest(ctx, module, raw)
switch {
case err == nil:
digests[dir] = digest
case errors.Is(err, errCannotBeAValue):
ix.logf("%s#%d: the manifest of %q cannot be kept: %v", f.Repository, f.Number, dir, err)
default:
return nil, err
}
}
return digests, nil
}
var errCannotBeAValue = errors.New("not a manifest value")
modules/mesh-delivery-indexer/internal/index/manifests_test.go
+102
View File
@@ -0,0 +1,102 @@
package index
import (
"context"
"os"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The manifests these tests carry are real files, copied unchanged into testdata: the module.json of modules of
// mesh-catalog at its main commit a67f01963f0d9ef0913a63720b89e573ca7a16ef (testdata/catalogue-a67f0196-<module>.module.json)
// and of build-agent from mesh-controller at its main commit f2d7bc94 (testdata/controller-f2d7bc94-build-agent.module.json).
func fixture(t *testing.T, name string) string {
t.Helper()
b, err := os.ReadFile("testdata/" + name + ".module.json")
if err != nil {
t.Fatal(err)
}
return string(b)
}
func catalogueManifest(t *testing.T, module string) string {
return fixture(t, "catalogue-a67f0196-"+module)
}
// avahiGoSum is the sha256 of modules/avahi/go.sum in mesh-catalog at a67f0196 (sha256sum of that file).
const avahiGoSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
const catalogueRepo = "novox/mesh-catalog"
func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memStore) {
t.Helper()
ix, mods := modulesIndexer(t, recs...)
manifests := newMem()
ix.Manifests = manifests
return ix, mods, manifests
}
func withManifests(f delivery.Forge, manifests map[string]string, locks map[string]map[string]string) delivery.Forge {
f.Manifests, f.Locks = manifests, locks
return f
}
func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1
avahi := catalogueManifest(t, "avahi")
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
digest := delivery.ManifestDigest([]byte(avahi))
raw, ok := manifests.vals[digest]
if !ok || len(manifests.vals) != 1 {
t.Fatalf("manifests holds %d keys, want only %s", len(manifests.vals), digest)
}
m, err := delivery.DecodeManifest(raw)
if err != nil || m.Module != "avahi" || m.JSON != avahi {
t.Errorf("manifest = %+v err %v, want avahi and the bytes of the file", m, err)
}
if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != digest {
t.Errorf("module = %+v (%v), want the manifest digest %s", mod, ok, digest)
}
}
func TestTheSameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2
avahi := catalogueManifest(t, "avahi")
ix, _, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
// A second merge, and a pull message carrying it, the digest being the same.
f.Merge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
if err := ix.Apply(context.Background(), f); err != nil {
t.Errorf("a second merge: %v", err)
}
pull := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
if err := ix.Apply(context.Background(), pull); err != nil {
t.Errorf("a pull message: %v", err)
}
if manifests.writes != 1 || len(manifests.vals) != 1 {
t.Errorf("manifests written %d times into %d keys, want once into one", manifests.writes, len(manifests.vals))
}
}
func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a manifest that is not JSON cannot be kept byte for byte)
var logs []string
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": "not json"}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if len(manifests.vals) != 0 || len(logs) == 0 {
t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs)
}
if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" {
t.Errorf("module = %+v (%v), want no manifest digest", mod, ok)
}
}
modules/mesh-delivery-indexer/internal/index/modules.go
+49 -3
View File
@@ -3,6 +3,7 @@ package index
import (
"context"
"fmt"
"reflect"
"sort"
"strings"
@@ -47,8 +48,14 @@ func fingerprintOf(k Known, trees map[string]string) (fp, none string) {
// applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and
// why), with the source revision that names the merge commit as the repository's head when the indexer has seen it
// (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already
// holds is not written again.
// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it
// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449,
// 1450); this release writes no fingerprint of the new form for any module.
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
digests, err := ix.keepManifests(ctx, f)
if err != nil {
return err
}
for _, k := range ix.Builds.Modules(f.Repository) {
key, err := delivery.ModulesKey(k.Module)
if err != nil {
@@ -56,10 +63,22 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
continue
}
fp, none := fingerprintOf(k, f.Trees)
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge)}
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path]}
// The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which
// names what is missing, from its manifest and lock hashes (inputs.go).
raw, has := f.Manifests[k.Path]
switch {
case !has:
next.None2 = None2NoManifest
case next.Manifest == "":
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
default:
in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx))
next.Inputs2, next.None2 = in.Kept, in.None2
}
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
if found {
if cur, err := delivery.DecodeModule(value); err == nil && delivery.SameModule(cur, withV(next)) {
if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
return nil, false, nil
}
}
@@ -73,4 +92,31 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
return nil
}
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
func (ix *Indexer) baseFP(ctx context.Context) baseFP {
return func(module string) (string, string) {
key, err := delivery.ModulesKey(module)
if err != nil {
return "", err.Error()
}
value, _, found, err := ix.Modules.Get(ctx, key)
if err != nil {
return "", "its record cannot be read: " + err.Error()
}
if !found {
return "", "the indexer holds no record of it"
}
cur, err := delivery.DecodeModule(value)
switch {
case err != nil:
return "", "its record cannot be read: " + err.Error()
case cur.FP2 != "":
return cur.FP2, ""
case cur.None2 != "":
return "", cur.None2
}
return "", "its record carries no fingerprint of the new form"
}
}
func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }
modules/mesh-delivery-indexer/internal/index/newform_test.go
+85
View File
@@ -0,0 +1,85 @@
package index
import (
"context"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go).
func wantAvahiFP(t *testing.T) string {
t.Helper()
fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
if none != "" {
t.Fatal(none)
}
return fp
}
func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1
avahi := catalogueManifest(t, "avahi")
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z",
`{"Module":"avahi","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "avahi")
if !ok {
t.Fatal("no module avahi")
}
todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum {
t.Errorf("module = %+v, want its manifest digest and inputs", m)
}
if m.None2 != "toolchain not yet reported" || m.FP2 != "" {
t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2)
}
if m.FP != todays || m.None != "" {
t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays)
}
}
func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) { // 1444/AC2
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "ca-trust", catalogueRepo, "modules/ca-trust", "main", "", "2026-10-19T09:00:00Z", aInputs))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/ca-trust": "newtree"}), map[string]string{"modules/ca-trust": catalogueManifest(t, "ca-trust")}, nil)
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.None2 != "builder's fingerprint not yet known" || m.Inputs2 == nil {
t.Errorf("module = %+v (%v), want none2 naming the builder's fingerprint", m, ok)
}
}
func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
t.Fatal(err)
}
m, ok := moduleOf(t, mods, "a")
if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" {
t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok)
}
if len(manifests.vals) != 0 {
t.Errorf("manifests holds %d keys", len(manifests.vals))
}
}
func TestTheSameMergeAppliedAgainWritesNothing(t *testing.T) { // 1444 (a value the module already holds is not written again, with inputs kept)
avahi := catalogueManifest(t, "avahi")
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
for i := 0; i < 2; i++ {
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
}
if mods.writes != 1 {
t.Errorf("modules written %d times, want once", mods.writes)
}
}
modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-audit-logger.module.json
Vendored
+69
View File
@@ -0,0 +1,69 @@
{
"module": "audit-logger",
"version": "1",
"slug": "audit",
"consumes": [
"**"
],
"build": {
"artifacts": [
{
"name": "code",
"kind": "bundle",
"language": "typescript",
"entrypoints": [
"index.js",
"tools/index.js"
],
"loads": [
"index.js",
"tools/index.js"
],
"env": {
"AUDIT_LOG": "${dir:trail}/audit.log",
"AUDIT_SPOOL": "${dir:spool}"
}
}
]
},
"data": {
"own": [
{
"id": "trail",
"path": "${dir:trail}",
"class": "valuable",
"why": "the audit trail, written nowhere else"
},
{
"id": "spool",
"path": "${dir:spool}",
"class": "valuable",
"why": "events the trail could not take yet; after the bus gives one up, the only copy"
}
]
},
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": ".",
"unknown": "program"
},
{
"id": "trail",
"type": "directory",
"mode": "0700",
"unknown": "program"
},
{
"id": "spool",
"type": "directory",
"mode": "0700",
"unknown": "program"
}
],
"capabilities": [
"container-runtime"
]
}
modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-avahi.module.json
Vendored
+43
View File
@@ -0,0 +1,43 @@
{
"module": "avahi",
"version": "1",
"capabilities": [
"package-manager",
"service-manager"
],
"tools": [
"avahi_status",
"avahi_browse",
"avahi_resolve",
"avahi_services"
],
"resources": [
{
"id": "package",
"type": "package",
"package": "avahi"
},
{
"id": "daemon",
"type": "service",
"unit": "avahi-daemon.service",
"state": "running",
"boot": "enabled"
}
],
"build": {
"artifacts": [
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/avahi-tools",
"binary": "avahi-tools",
"loads": [
"avahi-tools"
]
}
]
}
}
modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-ca-trust.module.json
Vendored
+66
View File
@@ -0,0 +1,66 @@
{
"module": "ca-trust",
"version": "1",
"slug": "catrust",
"capabilities": [
"service-manager"
],
"requires": [
"internal-acme-ca"
],
"seats": [
{
"name": "the-mesh-trust-anchor",
"scope": "node"
}
],
"claims": [
{
"name": "the-mesh-trust-anchor",
"scope": "node"
}
],
"resources": [
{
"id": "state",
"type": "directory",
"mode": "0700",
"place": ".",
"unknown": "program"
},
{
"id": "anchor",
"type": "file",
"path": "${dir:state}/anchor",
"mode": "0755",
"template": "files/anchor"
},
{
"id": "unit",
"type": "file",
"path": "/etc/systemd/system/mesh-ca-trust.service",
"mode": "0644",
"template": "files/mesh-ca-trust.service"
},
{
"id": "trust",
"type": "service",
"unit": "mesh-ca-trust.service",
"state": "running",
"boot": "enabled",
"restart-on": [
"anchor",
"unit"
]
}
],
"build": {
"artifacts": [
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-delivery-indexer/internal/index/testdata/catalogue-a67f0196-netcheck.module.json
Vendored
+57
View File
@@ -0,0 +1,57 @@
{
"module": "netcheck",
"version": "1",
"emits": [
"tunnel.endpoint-changed",
"tunnel.handshake-stale",
"tunnel.handshake-fresh"
],
"tools": [
"netcheck_tunnel",
"netcheck_tcp",
"netcheck_dns",
"netcheck_http",
"netcheck_listening"
],
"replaces": {
"netcheck_listening": [
"ss -lunt",
"ss -ltn",
"netstat -lnt"
],
"netcheck_dns": [
"dig",
"nslookup"
],
"netcheck_tunnel": [
"wg show",
"wg show mesh0 dump"
]
},
"build": {
"artifacts": [
{
"name": "tools-go",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/netcheck",
"binary": "netcheck",
"loads": [
"netcheck"
]
},
{
"name": "tools-typescript",
"kind": "bundle",
"language": "typescript",
"entrypoints": [
"tools/index.js"
],
"loads": [
"tools/index.js"
]
}
]
}
}
modules/mesh-delivery-indexer/internal/index/testdata/controller-f2d7bc94-build-agent.module.json
Vendored
+140
View File
@@ -0,0 +1,140 @@
{
"module": "build-agent",
"version": "1",
"slug": "agent",
"capabilities": [
"container-runtime"
],
"claims": [
{
"name": "node-build-agent",
"scope": "node",
"serves": ["current", "kill", "pause", "resume"]
}
],
"reads": [
"mesh-delivery-gate.pulls",
"mesh-delivery.pulls"
],
"requires": [
"artifact-store",
{
"name": "npm-package-registry",
"optional": true
}
],
"binds": {
"npm-package-registry": "${dir:mesh-state}/package-registry.json"
},
"secrets": {
"npm-package-registry": "${dir:mesh-state}/package-registry.secret"
},
"own-secrets": {
"broker": "${dir:mesh-state}/broker",
"genesis-package-registry": {
"path": "${dir:mesh-state}/genesis-package-registry.json",
"issued-by": "outside"
}
},
"data": {
"own": [
{
"id": "workspace",
"path": "${dir:workspace}",
"class": "cache",
"why": "a build's working copy, cloned again for every build"
},
{
"id": "judge-cache",
"path": "${dir:judge-cache}",
"class": "cache",
"why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)"
}
]
},
"resources": [
{
"id": "mesh-state",
"type": "directory",
"mode": "0700",
"place": "mesh"
},
{
"id": "workspace",
"type": "directory",
"mode": "0700"
},
{
"id": "judge-cache",
"type": "directory",
"mode": "0700"
},
{
"id": "agent-env",
"type": "file",
"path": "${dir:mesh-state}/build-agent.env",
"mode": "0600",
"template": "files/build-agent.env"
},
{
"id": "server",
"type": "container",
"name": "mesh-build-agent",
"artifact": "server",
"env-file": [
"${dir:mesh-state}/build-agent.env"
],
"volumes": [
"${dir:mesh-state}:/run/mesh:ro",
"${dir:workspace}:${dir:workspace}",
"${dir:judge-cache}:${dir:judge-cache}",
"/var/run/docker.sock:/var/run/docker.sock"
],
"restart-on": [
"agent-env"
],
"network": "host",
"health": {
"kind": "exec",
"command": "/usr/local/bin/mesh-builder ready",
"interval": "10s",
"timeout": "5s",
"looks": 3,
"grace": "60s"
},
"busy": {
"command": "/usr/local/bin/mesh-builder busy"
}
}
],
"build": {
"artifacts": [
{
"name": "server",
"kind": "image",
"from": "Dockerfile",
"compiles": "cmd/mesh-builder",
"context": {
"seat": "git",
"repository": "novox/mesh-controller",
"ref": "main"
}
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
],
"on": [
{
"arg": "GO_BASE",
"image": "golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c"
},
{
"arg": "ALPINE_BASE",
"image": "alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"
}
]
}
}
modules/mesh-delivery-indexer/manifest_test.go
+1 -1
View File
@@ -76,7 +76,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1
if m.Module != "mesh-delivery-indexer" {
t.Errorf("module = %q", m.Module)
}
if want := []state{{Name: "candidates", History: 10}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-controller.builds", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
modules/mesh-delivery-indexer/module.json
+1
View File
@@ -7,6 +7,7 @@
],
"state": [
{ "name": "candidates", "history": 10 },
{ "name": "manifests", "history": 1 },
{ "name": "modules", "history": 20 }
],
"reads": [