The bridge carries each module's manifest and lock hashes, and the indexer writes manifests by digest and each module's new-form inputs (design 47 R4c-2, issues 1448, 1449, 1450, 1444, 1451, 1458) #10
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -89,3 +90,34 @@ func treesVia(ctx context.Context, call caller) bridge.Trees {
|
||||
return a.Root, a.Trees, nil
|
||||
}
|
||||
}
|
||||
|
||||
// getFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file):
|
||||
// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text
|
||||
// (client.ts getFile). A path the forge does not have fails with `Gitea API <path>: 404 <body>` (client.ts request).
|
||||
const getFileTool = "gitea.gitea_get_file"
|
||||
|
||||
// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670
|
||||
// recognises it as /: 404 /).
|
||||
var notFound = regexp.MustCompile(`: 404 `)
|
||||
|
||||
// filesVia asks the forge for a file through call. A file the forge says is not there is bridge.ErrNotFound.
|
||||
func filesVia(ctx context.Context, call caller) bridge.Files {
|
||||
return func(owner, repo, path, ref string) ([]byte, error) {
|
||||
raw, err := call(ctx, getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
|
||||
if err != nil {
|
||||
if notFound.MatchString(err.Error()) {
|
||||
return nil, fmt.Errorf("%w: %v", bridge.ErrNotFound, err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var a struct {
|
||||
File *struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"file"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
|
||||
return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
|
||||
}
|
||||
return []byte(*a.File.Content), nil
|
||||
}
|
||||
}
|
||||
|
||||
+76
-7
@@ -4,6 +4,8 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -29,8 +31,8 @@ import (
|
||||
// bucket performed is the bridge's own (its state), and the five forge tools of the gate's acts are invoked (issue 1341, 1342).
|
||||
//
|
||||
// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal
|
||||
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls and
|
||||
// gitea.gitea_commit_trees, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
|
||||
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls,
|
||||
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
|
||||
// WirePattern). The consumer's durable is <node>_<module> on EVENTS (nats.go consumerDurable). The controller's own
|
||||
// user keeps all rights, as it has.
|
||||
const busConf = `
|
||||
@@ -59,6 +61,10 @@ accounts {
|
||||
"mesh.mod.gitea.call.gitea_open_pulls.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_commit_trees"
|
||||
"mesh.mod.gitea.tool.gitea_commit_trees.*"
|
||||
"mesh.mod.gitea.call.gitea_get_file.*.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_get_file.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_get_file"
|
||||
"mesh.mod.gitea.tool.gitea_get_file.*"
|
||||
"mesh.mod.gitea.tool.gitea_open_pulls"
|
||||
"mesh.mod.gitea.tool.gitea_open_pulls.*"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_performed.>"
|
||||
@@ -160,6 +166,21 @@ type bed struct {
|
||||
ctx context.Context
|
||||
done <-chan error
|
||||
reply func(string)
|
||||
// files sets what the stand-in forge answers to gitea_get_file from then on; it is asked with the tool's arguments.
|
||||
files func(func(owner, repo, path, ref string) (string, bool))
|
||||
}
|
||||
|
||||
// getFileReply is the reply envelope for gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile
|
||||
// answer it ({file: {path, ref, sha, size, content}}), and for a path the forge does not have the error client.ts request
|
||||
// writes: `Gitea API <path>: 404 <body>`.
|
||||
func getFileReply(path, ref, content string, found bool) []byte {
|
||||
if !found {
|
||||
b, _ := json.Marshal(map[string]any{"error": "Gitea API /repos/novox/mesh-delivery/contents/" + path + "?ref=" + ref + `: 404 {"message":"GetContentsOrList","url":"https://forge.example/api/swagger"}`})
|
||||
return b
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{"result": map[string]any{"file": map[string]any{"path": path, "ref": ref, "sha": "0123456789abcdef0123456789abcdef01234567",
|
||||
"size": len(content), "content": content}}, "node": "n0"})
|
||||
return b
|
||||
}
|
||||
|
||||
// startBed is startBridge with a step between the streams and the consumer: prepare writes what the streams hold before
|
||||
@@ -222,6 +243,26 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A stand-in answers the forge's gitea_get_file: by default a module.json of {} at every path and no other file.
|
||||
var fmu sync.Mutex
|
||||
filesOf := func(_, _, path, _ string) (string, bool) { return "{}", strings.HasSuffix(path, "module.json") }
|
||||
getFile, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_get_file", func(m *nats.Msg) {
|
||||
var a struct{ Owner, Repo, Path, Ref string }
|
||||
_ = json.Unmarshal(m.Data, &a)
|
||||
fmu.Lock()
|
||||
f := filesOf
|
||||
fmu.Unlock()
|
||||
content, found := f(a.Owner, a.Repo, a.Path, a.Ref)
|
||||
_ = m.Respond(getFileReply(a.Path, a.Ref, content, found))
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = getFile.Unsubscribe() })
|
||||
if err := ctl.Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
broker := filepath.Join(t.TempDir(), "broker")
|
||||
cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-forge-bridge", User: "n1.mesh-delivery-forge-bridge", Password: "pw"})
|
||||
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||
@@ -232,7 +273,8 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }()
|
||||
t.Cleanup(stop)
|
||||
return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() }}
|
||||
return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() },
|
||||
files: func(f func(owner, repo, path, ref string) (string, bool)) { fmu.Lock(); filesOf = f; fmu.Unlock() }}
|
||||
}
|
||||
|
||||
// TestBridgeWritesPullMessages starts the program the way the node does (its entry, run, with MESH_BROKER_FILE).
|
||||
@@ -270,7 +312,7 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "pull", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true}
|
||||
Action: "updated", Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
@@ -295,8 +337,31 @@ const merge = "89abcdef0123456789abcdef0123456789abcdef"
|
||||
// TestBridgeWritesMergedMessage has a stand-in answer the forge's gitea_commit_trees on its tool subject
|
||||
// (mesh-control internal/broker/nats.go invokedSubjects: mesh.mod.<module>.tool.<tool>), in the reply envelope of
|
||||
// mesh-sdk go/process/call.go ({result, error, node}).
|
||||
func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4
|
||||
ctl, js, ctx, _ := startBridge(t, `{"result":{"pulls":[]}}`)
|
||||
func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5
|
||||
bed := startBed(t, `{"result":{"pulls":[]}}`, nil)
|
||||
ctl, js, ctx := bed.ctl, bed.js, bed.ctx
|
||||
// The stand-in forge's files are two real files of this repository (this module's module.json and go.sum, read from the
|
||||
// working tree), in gitea_get_file's real answer shape: the root has a manifest only, modules/a a manifest and go.sum.
|
||||
manifest, err := os.ReadFile("../../module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sum, err := os.ReadFile("../../go.sum")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bed.files(func(_, _, path, ref string) (string, bool) {
|
||||
if ref != merge {
|
||||
t.Errorf("the forge was asked for %s at %s, want the merge commit %s", path, ref, merge)
|
||||
}
|
||||
switch path {
|
||||
case "module.json", "modules/a/module.json":
|
||||
return string(manifest), true
|
||||
case "modules/a/go.sum":
|
||||
return string(sum), true
|
||||
}
|
||||
return "", false
|
||||
})
|
||||
var asked chan []byte = make(chan []byte, 4)
|
||||
sub, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) {
|
||||
asked <- m.Data
|
||||
@@ -342,7 +407,9 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: "merged", Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head,
|
||||
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"}}
|
||||
Merge: merge, Root: "r0", Trees: map[string]string{"": "t0", "modules/a": "t1"},
|
||||
Manifests: map[string]string{"": string(manifest), "modules/a": string(manifest)},
|
||||
Locks: map[string]map[string]string{"modules/a": {"go.sum": hex.EncodeToString(sha256Sum(sum))}}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
@@ -637,3 +704,5 @@ func forgeMerged(t *testing.T, b *bed) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sha256Sum(b []byte) []byte { s := sha256.Sum256(b); return s[:] }
|
||||
|
||||
@@ -115,8 +115,8 @@ type publisher interface {
|
||||
// handle turns one event into its message on FORGE and acknowledges it. An event it cannot read is terminated, never
|
||||
// redelivered, and the log names its subject and why. A message it could not write stays unacknowledged: the server
|
||||
// delivers it again (the server's ack wait is the only clock).
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, logger *log.Logger) {
|
||||
msg, err := bridge.Convert(m.Subject(), m.Data(), trees)
|
||||
func handle(ctx context.Context, m delivered, pub publisher, trees bridge.Trees, files bridge.Files, logger *log.Logger) {
|
||||
msg, err := bridge.Convert(m.Subject(), m.Data(), trees, files)
|
||||
switch {
|
||||
case errors.Is(err, bridge.ErrRetry):
|
||||
logger.Printf("%v; heard again in %s", err, retryDelay)
|
||||
@@ -238,7 +238,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
// Bounded: the turn is held, and no sync or other event goes on without it.
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
handle(hctx, m, rec, treesVia(hctx, call), logger)
|
||||
handle(hctx, m, rec, treesVia(hctx, call), filesVia(hctx, call), logger)
|
||||
})
|
||||
},
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
|
||||
@@ -6,7 +6,9 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +16,8 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
|
||||
const head = "0123456789abcdef0123456789abcdef01234567"
|
||||
@@ -64,7 +68,7 @@ func body(m map[string]any) []byte {
|
||||
func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit part; the bus is the integration test)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" {
|
||||
t.Fatalf("published on %v", pub.subjects)
|
||||
}
|
||||
@@ -80,7 +84,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit
|
||||
func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{}
|
||||
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
@@ -95,7 +99,7 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5
|
||||
func TestUnwrittenMessageIsNotAcked(t *testing.T) { // 1235 (a failed write is redelivered by the server, not retried here)
|
||||
var logs bytes.Buffer
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")}
|
||||
handle(context.Background(), m, pub, nil, log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, nil, someFiles, log.New(&logs, "", 0))
|
||||
if m.acked != 0 || m.termd != 0 {
|
||||
t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd)
|
||||
}
|
||||
@@ -117,7 +121,7 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("the tree listing of novox/mesh-delivery was cut")
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
if len(pub.subjects) != 0 {
|
||||
t.Errorf("a message was written: %v", pub.subjects)
|
||||
}
|
||||
@@ -136,13 +140,19 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/
|
||||
m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{}
|
||||
var keys []string
|
||||
var args []any
|
||||
var mu sync.Mutex // the files are asked for concurrently (issue 1458)
|
||||
call := func(_ context.Context, key string, body any) (json.RawMessage, error) {
|
||||
mu.Lock()
|
||||
keys, args = append(keys, key), append(args, body)
|
||||
mu.Unlock()
|
||||
if key == "gitea.gitea_get_file" { // the manifests, 1448; the trees are what this test is about
|
||||
return json.RawMessage(`{"file":{"path":"p","sha":"0123","size":2,"content":"{}"}}`), nil
|
||||
}
|
||||
return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil
|
||||
}
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
if len(keys) != 1 || keys[0] != "gitea.gitea_commit_trees" {
|
||||
t.Fatalf("calls %v", keys)
|
||||
handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), log.New(&logs, "", 0))
|
||||
if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 {
|
||||
t.Fatalf("calls %v, want the trees once and first", keys)
|
||||
}
|
||||
if got, _ := json.Marshal(args[0]); string(got) != `{"owner":"novox","repo":"mesh-delivery","sha":"89abcdef0123456789abcdef0123456789abcdef"}` {
|
||||
t.Errorf("arguments %s", got)
|
||||
@@ -171,3 +181,52 @@ func TestAnAnswerWithoutARootIsNotUsed(t *testing.T) { // 1236 (a shape the tool
|
||||
t.Error("an answer that is not an object was taken")
|
||||
}
|
||||
}
|
||||
|
||||
// The answer of gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile write it, and the
|
||||
// error of a path the forge does not have as client.ts request writes it (`Gitea API ${path}: ${status} ${text}`).
|
||||
const getFileAnswer = `{"file":{"path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","sha":"0123","size":4,"content":"x y\n"}}`
|
||||
const getFile404 = "Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=89abcdef: 404 {\"message\":\"GetContentsOrList\",\"url\":\"https://forge.example/api/swagger\"}"
|
||||
|
||||
func TestFilesViaAsksByOwnerRepoPathRefAndTakesTheContent(t *testing.T) { // 1448/AC4
|
||||
var keys []string
|
||||
var args []any
|
||||
call := func(_ context.Context, key string, body any) (json.RawMessage, error) {
|
||||
keys, args = append(keys, key), append(args, body)
|
||||
return json.RawMessage(getFileAnswer), nil
|
||||
}
|
||||
got, err := filesVia(context.Background(), call)("novox", "mesh-delivery", "modules/a/go.sum", "89abcdef0123456789abcdef0123456789abcdef")
|
||||
if err != nil || string(got) != "x y\n" {
|
||||
t.Fatalf("got %q err %v", got, err)
|
||||
}
|
||||
if len(keys) != 1 || keys[0] != "gitea.gitea_get_file" {
|
||||
t.Fatalf("calls %v", keys)
|
||||
}
|
||||
if b, _ := json.Marshal(args[0]); string(b) != `{"owner":"novox","path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","repo":"mesh-delivery"}` {
|
||||
t.Errorf("arguments %s", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesViaTellsNotFoundFromOtherErrorsAndOtherShapes(t *testing.T) { // 1448/AC3
|
||||
call := func(err error, raw string) caller {
|
||||
return func(context.Context, string, any) (json.RawMessage, error) { return json.RawMessage(raw), err }
|
||||
}
|
||||
if _, err := filesVia(context.Background(), call(errors.New(getFile404), ""))("o", "r", "p", "s"); !errors.Is(err, bridge.ErrNotFound) {
|
||||
t.Errorf("a 404 is %v, want ErrNotFound", err)
|
||||
}
|
||||
if _, err := filesVia(context.Background(), call(errors.New("Gitea API /repos/o/r/contents/p: 502 bad gateway"), ""))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) {
|
||||
t.Errorf("a 502 is %v, want an error that is not ErrNotFound", err)
|
||||
}
|
||||
for _, raw := range []string{`{"file":null}`, `{}`, `[1]`, `{"file":{"path":"p"}}`} {
|
||||
if _, err := filesVia(context.Background(), call(nil, raw))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) {
|
||||
t.Errorf("the answer %s gave %v, want an error", raw, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// someFiles is a forge that has a module.json at every path and no other file, for the tests that are not about the files.
|
||||
func someFiles(_, _, path, _ string) ([]byte, error) {
|
||||
if strings.HasSuffix(path, "module.json") {
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
return nil, bridge.ErrNotFound
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9 h1:XL1dbtSleGAixg6FX4gEnSG/AJxRG/tWw6/7U1RuzHc=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019195003-4580d80a32d9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -2,13 +2,17 @@
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
@@ -34,6 +38,17 @@ const Trunk = "main"
|
||||
// (the root directory spelled ""), as the forge's gitea_commit_trees answers (issue 1234/AC4).
|
||||
type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error)
|
||||
|
||||
// ErrNotFound says the forge has no file at that path and ref (the forge's "404", issue 1448).
|
||||
var ErrNotFound = errors.New("the forge has no such file")
|
||||
|
||||
// Files asks the forge for one file's bytes at a ref (a commit), as the forge's gitea_get_file answers; a file the forge
|
||||
// says is not there is ErrNotFound, any other failure any other error (issue 1448).
|
||||
type Files func(owner, repo, path, ref string) ([]byte, error)
|
||||
|
||||
// LockFiles are the lock files read beside a module's manifest, by name relative to its directory (novox/hq ADR 0465
|
||||
// clause 1: go.sum for a Go module, package-lock.json for a Node one).
|
||||
var LockFiles = []string{"go.sum", "package-lock.json"}
|
||||
|
||||
// Message is a message for FORGE: the event name (forge.<owner>.<name>.<kind>) and its JSON body.
|
||||
type Message struct {
|
||||
Event string
|
||||
@@ -72,7 +87,7 @@ func unreadable(subject, why string, args ...any) error {
|
||||
// is an ErrUnreadable error naming the subject and why; an event it writes nothing for is ErrNotHandled. A merge into
|
||||
// the trunk asks trees once and becomes a merged message; a merge into any other branch is a closed pull message, as the
|
||||
// pull request left its group. A forge that cannot say is ErrRetry; a merge with no module.json is ErrNoModules.
|
||||
func Convert(subject string, data []byte, trees Trees) (Message, error) {
|
||||
func Convert(subject string, data []byte, trees Trees, files Files) (Message, error) {
|
||||
var action string
|
||||
switch subject {
|
||||
case UpdatedSubject:
|
||||
@@ -99,13 +114,17 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) {
|
||||
return Message{}, unreadable(subject, "no branch")
|
||||
}
|
||||
if subject == MergedSubject && p.Base == Trunk {
|
||||
return merged(subject, p, trees)
|
||||
return merged(subject, p, trees, files)
|
||||
}
|
||||
f := delivery.Forge{Kind: delivery.ForgePull, Repository: p.Owner + "/" + p.Repo, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Action: action}
|
||||
// The directories are the forge's word only when it said it could read them (module_dirs_said).
|
||||
if action == delivery.PullUpdated && p.ModuleDirsSaid {
|
||||
f.Dirs, f.DirsKnown = p.ModuleDirs, true
|
||||
var err error
|
||||
if f.Manifests, f.Locks, err = inputsOf(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
|
||||
}
|
||||
}
|
||||
event, err := delivery.ForgeEvent(f.Repository, delivery.ForgePull)
|
||||
if err != nil {
|
||||
@@ -118,8 +137,97 @@ func Convert(subject string, data []byte, trees Trees) (Message, error) {
|
||||
return Message{Event: event, Data: body}, nil
|
||||
}
|
||||
|
||||
// MaxReadsInFlight is how many files of the forge inputsOf asks for at once. A merge into a repository of many
|
||||
// modules (131 in the catalogue) reads three files for each, one after another inside the bound of a handled event
|
||||
// (30 s) that leaves about 76 ms a call; eight at once keeps the forge's load small and the read well inside the bound
|
||||
// (issue 1458).
|
||||
const MaxReadsInFlight = 8
|
||||
|
||||
// read is one file asked for: the path in the directory dir, and whether it is the module's manifest.
|
||||
type read struct {
|
||||
dir, path string
|
||||
manifest bool
|
||||
raw []byte
|
||||
err error
|
||||
}
|
||||
|
||||
// inputsOf reads, at ref, the module.json of each directory (the root spelled "") and the lock files beside it
|
||||
// (LockFiles): the manifests as the forge answered them and the lock files' hex sha256 by name. A manifest the forge
|
||||
// does not have is an error, since the directory was said to hold one; a lock file it does not have is left out; any other
|
||||
// failure is an error (issue 1448).
|
||||
//
|
||||
// The files are asked for with at most MaxReadsInFlight calls at once (issue 1458). The result does not depend on the
|
||||
// order the calls finish in, and the error is that of the first failing file in the order of dirs and LockFiles. After
|
||||
// a failure no further file is asked for, and inputsOf returns only when every call it started has returned.
|
||||
func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) {
|
||||
at := func(dir, name string) string {
|
||||
if dir == "" {
|
||||
return name
|
||||
}
|
||||
return dir + "/" + name
|
||||
}
|
||||
var reads []*read
|
||||
for _, dir := range dirs {
|
||||
reads = append(reads, &read{dir: dir, path: at(dir, "module.json"), manifest: true})
|
||||
for _, name := range LockFiles {
|
||||
reads = append(reads, &read{dir: dir, path: at(dir, name)})
|
||||
}
|
||||
}
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
failed bool
|
||||
slots = make(chan struct{}, MaxReadsInFlight)
|
||||
)
|
||||
for _, r := range reads {
|
||||
slots <- struct{}{}
|
||||
mu.Lock()
|
||||
stop := failed
|
||||
mu.Unlock()
|
||||
if stop {
|
||||
<-slots
|
||||
break
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(r *read) {
|
||||
defer func() { <-slots; wg.Done() }()
|
||||
r.raw, r.err = files(owner, repo, r.path, ref)
|
||||
if r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)) {
|
||||
mu.Lock()
|
||||
failed = true
|
||||
mu.Unlock()
|
||||
}
|
||||
}(r)
|
||||
}
|
||||
wg.Wait()
|
||||
manifests = map[string]string{}
|
||||
locks = map[string]map[string]string{}
|
||||
for _, r := range reads {
|
||||
switch {
|
||||
case r.err != nil && (r.manifest || !errors.Is(r.err, ErrNotFound)):
|
||||
return nil, nil, fmt.Errorf("%s: %w", r.path, r.err)
|
||||
case r.err != nil: // a lock file the forge does not have
|
||||
case r.manifest:
|
||||
manifests[r.dir] = string(r.raw)
|
||||
default:
|
||||
sum := sha256.Sum256(r.raw)
|
||||
if locks[r.dir] == nil {
|
||||
locks[r.dir] = map[string]string{}
|
||||
}
|
||||
locks[r.dir][path.Base(r.path)] = hex.EncodeToString(sum[:])
|
||||
}
|
||||
}
|
||||
if len(manifests) == 0 {
|
||||
manifests = nil
|
||||
}
|
||||
if len(locks) == 0 {
|
||||
locks = nil
|
||||
}
|
||||
return manifests, locks, nil
|
||||
}
|
||||
|
||||
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
|
||||
func merged(subject string, p pull, trees Trees) (Message, error) {
|
||||
func merged(subject string, p pull, trees Trees, files Files) (Message, error) {
|
||||
if !sha40.MatchString(p.MergeSHA) {
|
||||
return Message{}, unreadable(subject, "the merge commit %q is not 40 hex digits", p.MergeSHA)
|
||||
}
|
||||
@@ -131,12 +239,21 @@ func merged(subject string, p pull, trees Trees) (Message, error) {
|
||||
if len(dirs) == 0 {
|
||||
return Message{}, fmt.Errorf("%w: %s: %s at the merge commit %s", ErrNoModules, subject, repository, p.MergeSHA)
|
||||
}
|
||||
names := make([]string, 0, len(dirs))
|
||||
for dir := range dirs {
|
||||
names = append(names, dir)
|
||||
}
|
||||
slices.Sort(names)
|
||||
manifests, locks, err := inputsOf(p.Owner, p.Repo, p.MergeSHA, names, files)
|
||||
if err != nil {
|
||||
return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s: %v", ErrRetry, subject, repository, p.MergeSHA, err)
|
||||
}
|
||||
event, err := delivery.ForgeEvent(repository, delivery.ForgeMerged)
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
body, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs})
|
||||
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks})
|
||||
if err != nil {
|
||||
return Message{}, unreadable(subject, "%v", err)
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ const updatedSubject = "mesh.mod.gitea.event.pull.updated"
|
||||
const closedSubject = "mesh.mod.gitea.event.pull.closed"
|
||||
|
||||
func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; the bus is the integration test)
|
||||
msg, err := Convert(updatedSubject, updated(nil), nil)
|
||||
msg, err := Convert(updatedSubject, updated(nil), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -59,14 +59,14 @@ func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion;
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true}
|
||||
Head: head, Action: delivery.PullUpdated, Dirs: []string{"modules/a"}, DirsKnown: true, Manifests: map[string]string{"modules/a": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
msg, err := Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil)
|
||||
msg, err := Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -75,7 +75,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
t.Errorf("dirs = %v known = %v, want not known", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
// Said and empty is known and empty, not unknown.
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil)
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": []string{}, "module_dirs_said": true}), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -84,7 +84,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
t.Errorf("dirs = %v known = %v, want known and empty", f.Dirs, f.DirsKnown)
|
||||
}
|
||||
// Dirs listed without the forge saying it could read them are not trusted.
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil)
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs_said": nil}), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -94,7 +94,7 @@ func TestDirsNotSaidAreNotKnown(t *testing.T) { // 1235/AC3
|
||||
}
|
||||
|
||||
func TestClosedBecomesAClosedPullMessage(t *testing.T) { // 1235/AC4
|
||||
msg, err := Convert(closedSubject, closed(), nil)
|
||||
msg, err := Convert(closedSubject, closed(), nil, anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -130,7 +130,7 @@ func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5
|
||||
"closed no head": {closedSubject, mustDrop(closed(), "head_sha"), "head"},
|
||||
"closed no repo": {closedSubject, mustDrop(closed(), "repo"), "repo"},
|
||||
} {
|
||||
msg, err := Convert(c.subject, c.data, nil)
|
||||
msg, err := Convert(c.subject, c.data, nil, anyFiles)
|
||||
if err == nil {
|
||||
t.Errorf("%s: a message %q was made", name, msg.Event)
|
||||
continue
|
||||
@@ -157,7 +157,7 @@ func mustDrop(body []byte, key string) []byte {
|
||||
|
||||
func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235
|
||||
for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated"} {
|
||||
if _, err := Convert(s, updated(nil), nil); !errors.Is(err, ErrNotHandled) {
|
||||
if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) {
|
||||
t.Errorf("%s: err = %v, want ErrNotHandled", s, err)
|
||||
}
|
||||
}
|
||||
@@ -200,7 +200,7 @@ func treesOf(calls *[]call, root string, trees map[string]string, err error) Tre
|
||||
func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
|
||||
var calls []call
|
||||
answer := map[string]string{"modules/a": "t1", "modules/b": "t2"}
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil))
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", answer, nil), anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -212,7 +212,7 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12,
|
||||
Head: head, Merge: merge, Root: "r0", Trees: answer}
|
||||
Head: head, Merge: merge, Root: "r0", Trees: answer, Manifests: map[string]string{"modules/a": "{}", "modules/b": "{}"}}
|
||||
if !delivery.SameForge(f, want) {
|
||||
t.Errorf("message = %+v, want %+v", f, want)
|
||||
}
|
||||
@@ -223,7 +223,7 @@ func TestMergeIntoMainBecomesAMergedMessage(t *testing.T) { // 1236/AC1
|
||||
|
||||
func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil))
|
||||
msg, err := Convert(mergedSubject, mergedBody(map[string]any{"base": "feat/y"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -243,7 +243,7 @@ func TestMergeIntoAnotherBranchIsAClosedPull(t *testing.T) { // 1236/AC2
|
||||
|
||||
func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the conversion; the delay is in main_test)
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")))
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "", nil, errors.New("the listing for novox/mesh-delivery was cut")), anyFiles)
|
||||
if !errors.Is(err, ErrRetry) || msg.Data != nil {
|
||||
t.Fatalf("err = %v data = %s, want ErrRetry and no message", err, msg.Data)
|
||||
}
|
||||
@@ -256,7 +256,7 @@ func TestTreesRefusedWritesNothingAndSaysWhy(t *testing.T) { // 1236/AC3 (the co
|
||||
|
||||
func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (notes: a repository with no module.json)
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil))
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{}, nil), anyFiles)
|
||||
if !errors.Is(err, ErrNoModules) || msg.Data != nil {
|
||||
t.Errorf("err = %v data = %s", err, msg.Data)
|
||||
}
|
||||
@@ -264,7 +264,7 @@ func TestNoModuleJSONAnywhereWritesNoMergedMessage(t *testing.T) { // 1236 (note
|
||||
|
||||
func TestMergedWithoutAMergeCommitIsUnreadable(t *testing.T) { // 1236
|
||||
var calls []call
|
||||
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil))
|
||||
_, err := Convert(mergedSubject, mergedBody(map[string]any{"merge_commit_sha": "abc"}), treesOf(&calls, "r0", map[string]string{"a": "t"}, nil), anyFiles)
|
||||
if !errors.Is(err, ErrUnreadable) || len(calls) != 0 {
|
||||
t.Errorf("err = %v calls = %v", err, calls)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package bridge
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The bytes the forge answers in these tests are two real files of this repository, read from the working tree: the
|
||||
// module.json and the go.sum of modules/mesh-delivery-forge-bridge (two directories up). The forge's own answer shape
|
||||
// is mesh-catalog modules/gitea tools/index.ts gitea_get_file; the bridge sees only the bytes (issue 1448).
|
||||
func real(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile("../../" + name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func hexSum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
|
||||
|
||||
type fileCall struct{ owner, repo, path, ref string }
|
||||
|
||||
// forgeFiles answers as the forge does for a path: its bytes, ErrNotFound for a path in none, the error in errs.
|
||||
type forgeFiles struct {
|
||||
have map[string][]byte
|
||||
errs map[string]error
|
||||
mu sync.Mutex
|
||||
calls []fileCall
|
||||
}
|
||||
|
||||
func (f *forgeFiles) get(owner, repo, path, ref string) ([]byte, error) {
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, fileCall{owner, repo, path, ref})
|
||||
f.mu.Unlock()
|
||||
if err, ok := f.errs[path]; ok {
|
||||
return nil, err
|
||||
}
|
||||
if b, ok := f.have[path]; ok {
|
||||
return b, nil
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
// anyFiles is a forge that has a module.json at every path and no lock file, for tests that are not about the files.
|
||||
func anyFiles(owner, repo, path, ref string) ([]byte, error) {
|
||||
if strings.HasSuffix(path, "module.json") {
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
|
||||
manifest, sum := real(t, "module.json"), real(t, "go.sum")
|
||||
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": sum, "modules/b/module.json": []byte(`{"module":"b"}`)}}
|
||||
var calls []call
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1", "modules/b": "t2"}, nil), forge.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, err := delivery.DecodeForge(msg.Data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := map[string]string{"modules/a": string(manifest), "modules/b": `{"module":"b"}`}; !reflect.DeepEqual(f.Manifests, want) {
|
||||
t.Errorf("manifests = %v", f.Manifests)
|
||||
}
|
||||
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
|
||||
t.Errorf("locks = %v, want %v", f.Locks, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
|
||||
manifest := real(t, "module.json")
|
||||
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest}}
|
||||
msg, err := Convert(updatedSubject, updated(nil), nil, forge.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ := delivery.DecodeForge(msg.Data)
|
||||
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 0 {
|
||||
t.Errorf("manifests %v locks %v", f.Manifests, f.Locks)
|
||||
}
|
||||
if len(forge.calls) == 0 || forge.calls[0].ref != head {
|
||||
t.Errorf("calls %v, want the head %s", forge.calls, head)
|
||||
}
|
||||
// Dirs not known: neither field, and the forge is not asked.
|
||||
forge = &forgeFiles{}
|
||||
msg, err = Convert(updatedSubject, updated(map[string]any{"module_dirs": nil, "module_dirs_said": nil}), nil, forge.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f, _ = delivery.DecodeForge(msg.Data)
|
||||
if f.Manifests != nil || f.Locks != nil || len(forge.calls) != 0 {
|
||||
t.Errorf("manifests %v locks %v calls %v", f.Manifests, f.Locks, forge.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgeErrorOrAMissingManifestIsRetryAndALockNotFoundIsLeftOut(t *testing.T) { // 1448/AC3
|
||||
boom := errors.New("Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=x: 502 bad gateway")
|
||||
trees := func() Trees { var c []call; return treesOf(&c, "r0", map[string]string{"modules/a": "t1"}, nil) }
|
||||
for name, forge := range map[string]*forgeFiles{
|
||||
"a manifest error": {errs: map[string]error{"modules/a/module.json": boom}},
|
||||
"a manifest not found": {},
|
||||
"a lock file error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/go.sum": boom}},
|
||||
"a second lock error": {have: map[string][]byte{"modules/a/module.json": []byte("{}")}, errs: map[string]error{"modules/a/package-lock.json": boom}},
|
||||
} {
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
|
||||
if !errors.Is(err, ErrRetry) || msg.Data != nil {
|
||||
t.Errorf("%s: err = %v data = %s, want ErrRetry and no message", name, err, msg.Data)
|
||||
}
|
||||
_, err = Convert(updatedSubject, updated(nil), nil, forge.get)
|
||||
if !errors.Is(err, ErrRetry) {
|
||||
t.Errorf("%s (pull): err = %v, want ErrRetry", name, err)
|
||||
}
|
||||
}
|
||||
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
|
||||
msg, err := Convert(mergedSubject, mergedBody(nil), trees(), forge.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, _ := delivery.DecodeForge(msg.Data); f.Locks != nil {
|
||||
t.Errorf("locks = %v, want none for lock files the forge does not have", f.Locks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFilesAreAskedAtTheMergeCommitByOwnerRepoPathAndRef(t *testing.T) { // 1448/AC4 (the call's arguments; the manifest is in manifest_test)
|
||||
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": []byte("{}")}}
|
||||
var calls []call
|
||||
if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", map[string]string{"modules/a": "t1"}, nil), forge.get); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []fileCall{{"novox", "mesh-delivery", "modules/a/module.json", merge}, {"novox", "mesh-delivery", "modules/a/go.sum", merge},
|
||||
{"novox", "mesh-delivery", "modules/a/package-lock.json", merge}}
|
||||
sort.Slice(forge.calls, func(i, j int) bool { return forge.calls[i].path < forge.calls[j].path }) // they are asked for concurrently
|
||||
sort.Slice(want, func(i, j int) bool { return want[i].path < want[j].path })
|
||||
if !reflect.DeepEqual(forge.calls, want) {
|
||||
t.Errorf("calls = %v, want %v", forge.calls, want)
|
||||
}
|
||||
}
|
||||
|
||||
// slowForge answers like a forge that takes d a call: a module.json of the directory's name everywhere, a go.sum in
|
||||
// the directories whose number is even and in the root, and counts the calls in flight.
|
||||
type slowForge struct {
|
||||
d time.Duration
|
||||
mu sync.Mutex
|
||||
inFlight, most, started, n int
|
||||
failOn string
|
||||
finished int
|
||||
}
|
||||
|
||||
func (f *slowForge) get(_, _, path, _ string) ([]byte, error) {
|
||||
f.mu.Lock()
|
||||
f.inFlight++
|
||||
f.started++
|
||||
if f.inFlight > f.most {
|
||||
f.most = f.inFlight
|
||||
}
|
||||
f.mu.Unlock()
|
||||
time.Sleep(f.d)
|
||||
defer func() { f.mu.Lock(); f.inFlight--; f.finished++; f.mu.Unlock() }()
|
||||
switch {
|
||||
case path == f.failOn:
|
||||
return nil, errors.New("Gitea API " + path + ": 502 bad gateway")
|
||||
case strings.HasSuffix(path, "module.json"):
|
||||
return []byte(`{"module":"` + path + `"}`), nil
|
||||
case strings.HasSuffix(path, "go.sum") && (path == "go.sum" || path[len(path)-8]%2 == 0):
|
||||
return []byte("sum of " + path), nil
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
func manyDirs(n int) []string {
|
||||
dirs := []string{""}
|
||||
for i := 1; i < n; i++ {
|
||||
dirs = append(dirs, fmt.Sprintf("modules/m%02d", i))
|
||||
}
|
||||
return dirs
|
||||
}
|
||||
|
||||
func TestFortyDirectoriesAreReadEightAtATimeWithTheResultOfOneAtATime(t *testing.T) { // 1458/AC1
|
||||
dirs := manyDirs(40)
|
||||
slow := &slowForge{d: 10 * time.Millisecond}
|
||||
manifests, locks, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if slow.most > MaxReadsInFlight || slow.most < 2 {
|
||||
t.Errorf("at most %d calls were in flight, want more than one and at most %d", slow.most, MaxReadsInFlight)
|
||||
}
|
||||
if slow.started != 40*3 {
|
||||
t.Errorf("%d calls, want 120", slow.started)
|
||||
}
|
||||
// The same read one file at a time.
|
||||
wantM, wantL := map[string]string{}, map[string]map[string]string{}
|
||||
one := &slowForge{}
|
||||
for _, dir := range dirs {
|
||||
m, l, err := inputsOfOne(dir, one.get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantM[dir] = m
|
||||
if l != nil {
|
||||
wantL[dir] = l
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(manifests, wantM) || !reflect.DeepEqual(locks, wantL) {
|
||||
t.Errorf("the concurrent read differs from a one-at-a-time read: %d manifests %d locks, want %d and %d", len(manifests), len(locks), len(wantM), len(wantL))
|
||||
}
|
||||
if len(locks) == 0 || len(locks) == 40 {
|
||||
t.Errorf("%d directories with locks, the fixture should have some and not all", len(locks))
|
||||
}
|
||||
}
|
||||
|
||||
// inputsOfOne reads one directory with a single call at a time, as the bridge did before issue 1458.
|
||||
func inputsOfOne(dir string, files Files) (string, map[string]string, error) {
|
||||
at := func(name string) string {
|
||||
if dir == "" {
|
||||
return name
|
||||
}
|
||||
return dir + "/" + name
|
||||
}
|
||||
raw, err := files("o", "r", at("module.json"), "x")
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
var locks map[string]string
|
||||
for _, name := range LockFiles {
|
||||
b, err := files("o", "r", at(name), "x")
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if locks == nil {
|
||||
locks = map[string]string{}
|
||||
}
|
||||
s := sha256.Sum256(b)
|
||||
locks[name] = hex.EncodeToString(s[:])
|
||||
}
|
||||
return string(raw), locks, nil
|
||||
}
|
||||
|
||||
func TestOneFailingFileIsRetryAndEveryCallHasReturnedBeforeTheReadDoes(t *testing.T) { // 1458/AC2
|
||||
dirs := manyDirs(40)
|
||||
slow := &slowForge{d: 10 * time.Millisecond, failOn: "modules/m05/go.sum"}
|
||||
_, _, err := inputsOf("novox", "mesh-catalog", merge, dirs, slow.get)
|
||||
if err == nil || !strings.Contains(err.Error(), "modules/m05/go.sum") || !strings.Contains(err.Error(), "502") {
|
||||
t.Fatalf("err = %v, want the failing file named", err)
|
||||
}
|
||||
slow.mu.Lock()
|
||||
defer slow.mu.Unlock()
|
||||
if slow.inFlight != 0 || slow.finished != slow.started {
|
||||
t.Errorf("%d calls in flight, %d of %d finished, when the read returned", slow.inFlight, slow.finished, slow.started)
|
||||
}
|
||||
if slow.started >= 40*3 {
|
||||
t.Errorf("%d calls were made, want the read to stop asking after the failure", slow.started)
|
||||
}
|
||||
// Through Convert it is ErrRetry.
|
||||
var calls []call
|
||||
dirsMap := map[string]string{}
|
||||
for _, d := range dirs[1:] {
|
||||
dirsMap[d] = "t"
|
||||
}
|
||||
slow2 := &slowForge{failOn: "modules/m05/module.json"}
|
||||
if _, err := Convert(mergedSubject, mergedBody(nil), treesOf(&calls, "r0", dirsMap, nil), slow2.get); !errors.Is(err, ErrRetry) {
|
||||
t.Errorf("err = %v, want ErrRetry", err)
|
||||
}
|
||||
}
|
||||
@@ -93,7 +93,7 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
|
||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||
}
|
||||
if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view",
|
||||
"gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
|
||||
"gitea.gitea_get_file", "gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||
|
||||
@@ -25,7 +25,8 @@
|
||||
"gitea.gitea_note_append",
|
||||
"gitea.gitea_add_comment",
|
||||
"gitea.gitea_open_pulls",
|
||||
"gitea.gitea_commit_trees"
|
||||
"gitea.gitea_commit_trees",
|
||||
"gitea.gitea_get_file"
|
||||
],
|
||||
"own-secrets": {
|
||||
"broker": "${dir:state}/broker"
|
||||
|
||||
@@ -145,6 +145,10 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) (
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The bucket of the manifests, made by the controller from the manifest's state entry manifests (history 1, issue 1449).
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket, History: delivery.ManifestsHistory}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuildsBucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -203,3 +207,61 @@ func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9
|
||||
t.Errorf("module = %+v within 5 s, want fp %s", m, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIndexerKeepsManifestsAtAMerge: a merged message of mesh-catalog carrying the real manifest of the module avahi and the
|
||||
// hash of its go.sum (the bridge's Manifests and Locks, issue 1448) makes, within 5 s, modules.avahi with its manifest digest,
|
||||
// its inputs and the reason it has no fingerprint of the new form, and manifests.<digest> with the manifest byte for byte.
|
||||
func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4
|
||||
// The manifest is avahi's module.json from mesh-catalog at a67f01963f0d9ef0913a63720b89e573ca7a16ef (copied into the
|
||||
// index package's testdata); the lock hash is the sha256 of modules/avahi/go.sum there.
|
||||
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||
const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
|
||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
|
||||
rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
|
||||
if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301,
|
||||
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||
Locks: map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var m delivery.Module
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := modules.Get(ctx, "avahi"); err == nil {
|
||||
if m, err = delivery.DecodeModule(e.Value()); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
digest := delivery.ManifestDigest(manifest)
|
||||
if m.Manifest != digest || m.None2 != "toolchain not yet reported" || m.FP2 != "" || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" ||
|
||||
m.Inputs2.Locks["tools"] != goSum {
|
||||
t.Errorf("module = %+v within 5 s, want its manifest digest %s, its inputs and none2 'toolchain not yet reported'", m, digest)
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e, err := kv.Get(ctx, digest)
|
||||
if err != nil {
|
||||
t.Fatalf("manifests.%s: %v", digest, err)
|
||||
}
|
||||
if got, err := delivery.DecodeManifest(e.Value()); err != nil || got.Module != "avahi" || got.JSON != string(manifest) {
|
||||
t.Errorf("manifest = %+v err %v, want avahi and the file's bytes", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,6 +251,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
manifests, err := bucket(delivery.ManifestsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
builds, err := bucket(delivery.BuildsBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -270,7 +274,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
errs := make(chan error, 1)
|
||||
go func() { errs <- follow(ctx, w, each) }()
|
||||
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Builds: known, Logf: logger.Printf}
|
||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf}
|
||||
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||
|
||||
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
)
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337 h1:cZyFnvDiBpnNx8ifSUG9WA61OHsn7PC2ml2s4eD7AkA=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019203925-84a764372337/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a h1:Vjk2Kq+g/Q86N8CCHq3OatJIJ0JSy6Kh980ekGxNLmE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019223929-58373d65b47a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
func manifestPull(t *testing.T, number int, head, manifest string) delivery.Forge {
|
||||
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: number, Head: head,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true}
|
||||
if manifest != "" {
|
||||
f.Manifests = map[string]string{"modules/avahi": manifest}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
func candidateOf(t *testing.T, st *memStore) delivery.Candidate {
|
||||
t.Helper()
|
||||
key, err := delivery.CandidatesKey("feat/x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, ok := st.vals[key]
|
||||
if !ok {
|
||||
t.Fatalf("candidates holds %d keys, none %s", len(st.vals), key)
|
||||
}
|
||||
c, err := delivery.DecodeCandidate(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestACandidateGivesEachTouchedModuleItsManifestAndTheReasonItHasNoPreview(t *testing.T) { // 1451/AC1
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := candidateOf(t, ix.Store.(*memStore)) // DecodeCandidate has run Candidate.check
|
||||
want := delivery.CandidateModule{Manifest: delivery.ManifestDigest([]byte(avahi)), None2: "no merge preview is computed"}
|
||||
if len(c.Pulls) != 1 || len(c.Pulls[0].Fps) != 1 || c.Pulls[0].Fps["avahi"] != want {
|
||||
t.Fatalf("pulls = %+v, want Fps[avahi] = %+v", c.Pulls, want)
|
||||
}
|
||||
if c.FPSet == "" || c.FPSet != delivery.FingerprintSet(c.Pulls) {
|
||||
t.Errorf("fpset = %q, want %q", c.FPSet, delivery.FingerprintSet(c.Pulls))
|
||||
}
|
||||
// The same pull request heard again without its directories (the head unchanged) keeps what it had.
|
||||
again := manifestPull(t, 301, h1, "")
|
||||
again.Dirs, again.DirsKnown = nil, false
|
||||
if err := ix.Apply(context.Background(), again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if c2 := candidateOf(t, ix.Store.(*memStore)); c2.Pulls[0].Fps["avahi"] != want || c2.FPSet != c.FPSet {
|
||||
t.Errorf("after a message without directories: %+v fpset %q", c2.Pulls[0].Fps, c2.FPSet)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPullWithoutManifestsWritesTheCandidateAsToday(t *testing.T) { // 1451/AC2
|
||||
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := candidateOf(t, ix.Store.(*memStore))
|
||||
if len(c.Pulls) != 1 || c.Pulls[0].Fps != nil || c.FPSet != "" || len(c.Pulls[0].Modules) != 1 || c.Pulls[0].Modules[0] != "avahi" {
|
||||
t.Errorf("candidate = %+v, want the module avahi and no Fps and no fpset", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoPullsGivingOneModuleDifferentManifestsAreRefusedNamingBoth(t *testing.T) { // 1451/AC3
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
// The second pull request carries another real manifest (ca-trust's, mesh-catalog a67f0196) for the same directory.
|
||||
other := catalogueManifest(t, "ca-trust")
|
||||
ix, _, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), manifestPull(t, 301, h1, avahi)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := candidateOf(t, ix.Store.(*memStore))
|
||||
writes := ix.Store.(*memStore).writes
|
||||
err := ix.Apply(context.Background(), manifestPull(t, 302, h2, other))
|
||||
if !errors.Is(err, ErrUnreadable) {
|
||||
t.Fatalf("err = %v, want a refusal", err)
|
||||
}
|
||||
for _, w := range []string{catalogueRepo + "#301", catalogueRepo + "#302", `"avahi"`} {
|
||||
if !strings.Contains(err.Error(), w) {
|
||||
t.Errorf("the refusal %q does not name %s", err, w)
|
||||
}
|
||||
}
|
||||
// Candidate.check refuses the group too, which is what the refusal stands for.
|
||||
if _, encErr := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat.x", HeadsHash: before.HeadsHash, Pulls: []delivery.CandidatePull{before.Pulls[0], {
|
||||
Head: delivery.Head{Repository: catalogueRepo, Number: 302, Head: h2}, Modules: []string{"avahi"},
|
||||
Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(other)), None2: NoneNoPreview}}}}}); encErr == nil {
|
||||
t.Error("the candidate check accepts two manifests for one module")
|
||||
}
|
||||
if ix.Store.(*memStore).writes != writes {
|
||||
t.Error("a partial write was made")
|
||||
}
|
||||
if after := candidateOf(t, ix.Store.(*memStore)); len(after.Pulls) != 1 || after.FPSet != before.FPSet {
|
||||
t.Errorf("the candidate changed: %+v", after)
|
||||
}
|
||||
}
|
||||
@@ -43,8 +43,10 @@ type Indexer struct {
|
||||
Store Store // candidates
|
||||
// Modules is the modules bucket; nil writes no fingerprints.
|
||||
Modules Store
|
||||
Builds *Builds
|
||||
Logf func(format string, args ...any)
|
||||
// Manifests is the manifests bucket; nil writes no manifests (issue 1449).
|
||||
Manifests Store
|
||||
Builds *Builds
|
||||
Logf func(format string, args ...any)
|
||||
|
||||
mu sync.Mutex // guards source, which the source watch and the forge messages both reach
|
||||
source sourceNow
|
||||
@@ -72,6 +74,13 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var digests map[string]string
|
||||
if f.Kind == delivery.ForgePull {
|
||||
var err error
|
||||
if digests, err = ix.keepManifests(ctx, f); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if trunkLike[f.Branch] {
|
||||
ix.logf("%s#%d: its branch %q is no delivery group, so no candidate", f.Repository, f.Number, f.Branch)
|
||||
return nil
|
||||
@@ -89,7 +98,12 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
|
||||
}
|
||||
cur = &c
|
||||
}
|
||||
next := ix.next(cur, f)
|
||||
next := ix.next(cur, f, digests)
|
||||
if next != nil {
|
||||
if err := ix.refuseDisagreement(*next); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case next == nil && cur == nil, next != nil && cur != nil && delivery.SameCandidate(*cur, *next):
|
||||
return nil, false, nil
|
||||
@@ -185,7 +199,12 @@ func write(ctx context.Context, st Store, key string, compute func(value []byte,
|
||||
}
|
||||
|
||||
// next is the group after the message; nil when the group is left with no pull request.
|
||||
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Candidate {
|
||||
//
|
||||
// A pull message that carries manifests (issue 1448) gives each module it touches, in the pull request's Fps, the digest of
|
||||
// its manifest and why it has no fingerprint of the new form (NoneNoPreview), and the candidate its FPSet (issue 1451); a
|
||||
// pull request whose head is held again without its directories keeps the Fps it had. digests are the manifests' digests by
|
||||
// directory.
|
||||
func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge, digests map[string]string) *delivery.Candidate {
|
||||
var pulls []delivery.CandidatePull
|
||||
var base uint64
|
||||
if cur != nil {
|
||||
@@ -202,13 +221,20 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can
|
||||
switch {
|
||||
case f.DirsKnown:
|
||||
for _, dir := range f.Dirs {
|
||||
if m := ix.Builds.ModuleOf(f.Repository, dir); !slices.Contains(entry.Modules, m) {
|
||||
m := ix.Builds.ModuleOf(f.Repository, dir)
|
||||
if !slices.Contains(entry.Modules, m) {
|
||||
entry.Modules = append(entry.Modules, m)
|
||||
}
|
||||
if digest, ok := digests[dir]; ok {
|
||||
if entry.Fps == nil {
|
||||
entry.Fps = map[string]delivery.CandidateModule{}
|
||||
}
|
||||
entry.Fps[m] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview}
|
||||
}
|
||||
}
|
||||
case at >= 0 && pulls[at].Head.Head == f.Head:
|
||||
// Its directories are not known now, for the head the group already holds: what it knew stands.
|
||||
entry.Modules = pulls[at].Modules
|
||||
entry.Modules, entry.Fps = pulls[at].Modules, pulls[at].Fps
|
||||
}
|
||||
if at >= 0 {
|
||||
pulls[at] = entry
|
||||
@@ -230,5 +256,32 @@ func (ix *Indexer) next(cur *delivery.Candidate, f delivery.Forge) *delivery.Can
|
||||
heads[i] = p.Head
|
||||
}
|
||||
hash, _ := delivery.HeadsHash(heads)
|
||||
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls}
|
||||
return &delivery.Candidate{V: delivery.Version, Group: f.Branch, Base: base, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
|
||||
}
|
||||
|
||||
// NoneNoPreview is why a candidate's touched module has no fingerprint of the new form: the merge preview is not
|
||||
// computed in this release (novox/hq ADR 0465 clause 3; the checker comes with issue 1445).
|
||||
const NoneNoPreview = "no merge preview is computed"
|
||||
|
||||
// refuseDisagreement refuses a candidate in which two pull requests give one module different manifests, as the
|
||||
// candidate's own check does (mesh-sdk Candidate.check), naming both pull requests so the log says which; nothing is
|
||||
// written, and the message is dropped as unreadable and said in the log (main.go handle): the next message of either pull
|
||||
// request is judged again.
|
||||
func (ix *Indexer) refuseDisagreement(c delivery.Candidate) error {
|
||||
first := map[string]delivery.CandidatePull{}
|
||||
for _, p := range c.Pulls {
|
||||
for module, m := range p.Fps {
|
||||
before, ok := first[module]
|
||||
if !ok {
|
||||
first[module] = p
|
||||
continue
|
||||
}
|
||||
if before.Fps[module] != m {
|
||||
err := fmt.Errorf("%w: candidates.%s: %s#%d and %s#%d give the module %q different manifests (%s and %s), so the group is not written",
|
||||
ErrUnreadable, c.Group, before.Repository, before.Number, p.Repository, p.Number, module, before.Fps[module].Manifest, m.Manifest)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The reasons a module has no fingerprint of the new form while the indexer writes none for any module (novox/hq
|
||||
// ADR 0465 clause 2; issue 1444). Toolchains missing is delivery.ToolchainNoReport.
|
||||
const (
|
||||
// None2NoManifest is for a module whose directory the merged message carries no manifest of (an older bridge).
|
||||
None2NoManifest = "the message carries no manifest"
|
||||
// None2Builder is for a module with no artifact of a toolchain: only the builder's fingerprint is missing.
|
||||
None2Builder = "builder's fingerprint not yet known"
|
||||
// None2Context is for a module an artifact of which reads another repository's tree (today's form gives it none
|
||||
// either, NoneContext).
|
||||
None2Context = "context not pinned"
|
||||
)
|
||||
|
||||
// lockFile is the lock file of a language's artifact that resolves packages when it is built, by name in the module's
|
||||
// directory (the bridge reads these, internal/bridge LockFiles; issue 1450).
|
||||
var lockFile = map[string]string{"go": "go.sum", "typescript": "package-lock.json"}
|
||||
|
||||
// moduleInputs is what a module's manifest and lock hashes give the indexer at a merge (issue 1450).
|
||||
type moduleInputs struct {
|
||||
// Kept is the new-form inputs the merge gave the module; nil when the manifest could not be read or gave none.
|
||||
Kept *delivery.KeptInputs2
|
||||
// Languages is the language of each artifact that has a toolchain (a bundle or a package of go or typescript), by
|
||||
// artifact name. The toolchain versions are joined to them when the build modules report them (issue 1444, the
|
||||
// watching half); the SDK's KeptInputs2 keeps no languages, so they are read from the manifest again then.
|
||||
Languages map[string]string
|
||||
// None2 is why the module has no fingerprint of the new form.
|
||||
None2 string
|
||||
}
|
||||
|
||||
// baseFP answers a base module's new-form fingerprint, or why it has none.
|
||||
type baseFP func(module string) (fp, none string)
|
||||
|
||||
// inputsOf turns the manifest of a module (raw, as the forge answered it), the tree of its directory and the lock
|
||||
// hashes of that directory (lock file name to hash) into its new-form inputs. The manifest is read by the catalogue's
|
||||
// parser; a manifest it refuses, or one that is of another module, leaves Kept nil and says why.
|
||||
//
|
||||
// The rules are the controller's builder (mesh-controller internal/builder/builder.go): a bundle or a package of a
|
||||
// language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages,
|
||||
// resolves packages when it is built (wantsPackages, :1017), so it has an entry in Locks, with the hash of its
|
||||
// language's lock file or an empty hash when none was carried (the fingerprint is then none, "no lock"); an archive,
|
||||
// an image and an upstream resolve nothing and have no toolchain. A base is a build.on entry that names a module.
|
||||
func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs {
|
||||
m, err := catalogue.ParseManifest([]byte(raw))
|
||||
if err != nil {
|
||||
return moduleInputs{None2: "the manifest cannot be read: " + err.Error()}
|
||||
}
|
||||
if m.Module != module {
|
||||
return moduleInputs{None2: fmt.Sprintf("the manifest is of module %q, not %q", m.Module, module)}
|
||||
}
|
||||
if tree == "" {
|
||||
return moduleInputs{None2: "the merge names no tree for the module's directory"}
|
||||
}
|
||||
kept := delivery.KeptInputs2{Tree: tree}
|
||||
langs := map[string]string{}
|
||||
contexts := false
|
||||
var missing []string
|
||||
if m.Build != nil {
|
||||
for _, a := range m.Build.Artifacts {
|
||||
if a.Context != nil {
|
||||
contexts = true
|
||||
}
|
||||
if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactPackage {
|
||||
continue
|
||||
}
|
||||
name, ok := lockFile[a.Language]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
langs[a.Name] = a.Language
|
||||
if kept.Locks == nil {
|
||||
kept.Locks = map[string]string{}
|
||||
}
|
||||
kept.Locks[a.Name] = locks[name]
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, on := range m.Build.On {
|
||||
if on.Module == "" || seen[on.Module] {
|
||||
continue
|
||||
}
|
||||
seen[on.Module] = true
|
||||
if fp, none := base(on.Module); fp != "" {
|
||||
kept.Bases = append(kept.Bases, fp)
|
||||
} else {
|
||||
missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", on.Module, none))
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(kept.Bases)
|
||||
sort.Strings(missing)
|
||||
out := moduleInputs{Kept: &kept}
|
||||
if len(langs) > 0 {
|
||||
out.Languages = langs
|
||||
}
|
||||
switch {
|
||||
case contexts:
|
||||
out.None2 = None2Context
|
||||
case len(missing) > 0:
|
||||
out.None2 = strings.Join(missing, "; ")
|
||||
case len(langs) > 0:
|
||||
out.None2 = delivery.ToolchainNoReport
|
||||
default:
|
||||
out.None2 = None2Builder
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// aBase is the new-form fingerprint of a base module, the one SourceFingerprint2 gives for the inputs of avahi at a tree.
|
||||
func aBase(t *testing.T) string {
|
||||
t.Helper()
|
||||
fp, none := delivery.SourceFingerprint2(delivery.SourceInputs2{Module: "mesh-tools", Tree: "basetree"})
|
||||
if none != "" {
|
||||
t.Fatal(none)
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
func noBases(string) (string, string) { return "", "none are held in this test" }
|
||||
|
||||
func TestAGoModuleKeepsItsTreeAndTheHashOfItsLockFile(t *testing.T) { // 1450/AC1
|
||||
in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, noBases)
|
||||
want := &delivery.KeptInputs2{Tree: "avahitree", Locks: map[string]string{"tools": avahiGoSum}}
|
||||
if !reflect.DeepEqual(in.Kept, want) {
|
||||
t.Errorf("kept = %+v, want %+v", in.Kept, want)
|
||||
}
|
||||
if !reflect.DeepEqual(in.Languages, map[string]string{"tools": "go"}) {
|
||||
t.Errorf("languages = %v, want tools: go", in.Languages)
|
||||
}
|
||||
// Through the indexer: the merge's tree of the module's directory and the lock hashes of that directory, kept in modules.avahi.
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "avahitree", "modules/other": "x"}),
|
||||
map[string]string{"modules/avahi": catalogueManifest(t, "avahi")}, map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "avahi")
|
||||
if !ok || !reflect.DeepEqual(m.Inputs2, want) || m.Manifest != delivery.ManifestDigest([]byte(catalogueManifest(t, "avahi"))) {
|
||||
t.Errorf("module = %+v (%v), want inputs %+v and the manifest digest", m, ok, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnArtifactThatResolvesPackagesAndHasNoLockFileHasAnEmptyHash(t *testing.T) { // 1450/AC2
|
||||
in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", nil, noBases)
|
||||
if in.Kept == nil || !reflect.DeepEqual(in.Kept.Locks, map[string]string{"tools": ""}) {
|
||||
t.Fatalf("kept = %+v, want Locks {tools: \"\"}", in.Kept)
|
||||
}
|
||||
// The fingerprint of such inputs is none, "no lock" (mesh-sdk SourceFingerprint2).
|
||||
if fp, none := delivery.SourceFingerprint2(in.Kept.With("avahi", nil)); fp != "" || none != "no lock" {
|
||||
t.Errorf("fingerprint = %q none %q, want none 'no lock'", fp, none)
|
||||
}
|
||||
// A TypeScript bundle's lock file is package-lock.json, not go.sum.
|
||||
ts := inputsOf("audit-logger", catalogueManifest(t, "audit-logger"), "t", map[string]string{"go.sum": "wrong", "package-lock.json": avahiGoSum}, noBases)
|
||||
if ts.Kept == nil || !reflect.DeepEqual(ts.Kept.Locks, map[string]string{"code": avahiGoSum}) || !reflect.DeepEqual(ts.Languages, map[string]string{"code": "typescript"}) {
|
||||
t.Errorf("typescript: kept %+v languages %v", ts.Kept, ts.Languages)
|
||||
}
|
||||
// netcheck has a Go and a TypeScript bundle: each takes its own language's lock file.
|
||||
both := inputsOf("netcheck", catalogueManifest(t, "netcheck"), "t", map[string]string{"go.sum": "g", "package-lock.json": "p"}, noBases)
|
||||
if both.Kept == nil || !reflect.DeepEqual(both.Kept.Locks, map[string]string{"tools-go": "g", "tools-typescript": "p"}) {
|
||||
t.Errorf("netcheck: kept %+v", both.Kept)
|
||||
}
|
||||
}
|
||||
|
||||
// aModuleWithABase is avahi's manifest (mesh-catalog a67f0196) with the one build.on entry that names a module as the
|
||||
// controller's own test has it (mesh-controller internal/builder/standing_on_test.go:22): no manifest of the sources names
|
||||
// a module as a base, only vendors' images.
|
||||
func aModuleWithABase(t *testing.T) string {
|
||||
t.Helper()
|
||||
raw := catalogueManifest(t, "avahi")
|
||||
with := strings.Replace(raw, `"build": {`, `"build": {
|
||||
"on": [{"arg": "RUNTIME_BASE", "module": "mesh-tools", "artifact": "runtime"}],`, 1)
|
||||
if with == raw {
|
||||
t.Fatal("avahi's manifest has no build")
|
||||
}
|
||||
return with
|
||||
}
|
||||
|
||||
func TestABaseEntersByTheFingerprintOfItsModuleOrNamesItsAbsence(t *testing.T) { // 1450/AC3
|
||||
fp := aBase(t)
|
||||
has := func(module string) (string, string) {
|
||||
if module != "mesh-tools" {
|
||||
t.Errorf("asked for the base %q", module)
|
||||
}
|
||||
return fp, ""
|
||||
}
|
||||
in := inputsOf("avahi", aModuleWithABase(t), "t", map[string]string{"go.sum": avahiGoSum}, has)
|
||||
if in.Kept == nil || !reflect.DeepEqual(in.Kept.Bases, []string{fp}) {
|
||||
t.Errorf("bases = %+v, want [%s]", in.Kept, fp)
|
||||
}
|
||||
none := inputsOf("avahi", aModuleWithABase(t), "t", map[string]string{"go.sum": avahiGoSum}, noBases)
|
||||
if none.Kept == nil || len(none.Kept.Bases) != 0 || !strings.Contains(none.None2, "mesh-tools") || !strings.Contains(none.None2, "base") {
|
||||
t.Errorf("kept %+v none2 %q, want no bases and none2 naming the base mesh-tools", none.Kept, none.None2)
|
||||
}
|
||||
// Through the indexer: modules.mesh-tools holds a fingerprint of the new form (a record written by a later release).
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
mods.put("mesh-tools", mustEncode(t, delivery.Module{Repository: catalogueRepo, Commit: merge, None: "x", FP2: fp, Manifest: strings.Repeat("a", 64)}))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t"}), map[string]string{"modules/avahi": aModuleWithABase(t)}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, _ := moduleOf(t, mods, "avahi"); m.Inputs2 == nil || !reflect.DeepEqual(m.Inputs2.Bases, []string{fp}) {
|
||||
t.Errorf("module = %+v, want the base's fingerprint", m)
|
||||
}
|
||||
}
|
||||
|
||||
func mustEncode(t *testing.T, m delivery.Module) []byte {
|
||||
t.Helper()
|
||||
b, err := delivery.EncodeModule(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestAnArtifactOfNoToolchainHasNoLanguageAndNoLock(t *testing.T) { // 1450/AC4
|
||||
in := inputsOf("ca-trust", catalogueManifest(t, "ca-trust"), "t", map[string]string{"go.sum": avahiGoSum}, noBases)
|
||||
if in.Kept == nil || len(in.Kept.Locks) != 0 || len(in.Languages) != 0 {
|
||||
t.Errorf("kept %+v languages %v, want neither locks nor languages for an archive", in.Kept, in.Languages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAManifestTheParserRefusesLeavesNoInputsAndSaysWhy(t *testing.T) { // 1450/AC5
|
||||
for name, raw := range map[string]string{
|
||||
"not json": "not json",
|
||||
"unknown field": strings.Replace(catalogueManifest(t, "avahi"), `"version": "1",`, `"version": "1", "scope": "everything",`, 1),
|
||||
} {
|
||||
in := inputsOf("avahi", raw, "t", nil, noBases)
|
||||
if in.Kept != nil || !strings.HasPrefix(in.None2, "the manifest cannot be read: ") || len(in.None2) <= len("the manifest cannot be read: ") {
|
||||
t.Errorf("%s: kept %+v none2 %q, want nil and 'the manifest cannot be read: <error>'", name, in.Kept, in.None2)
|
||||
}
|
||||
}
|
||||
if in := inputsOf("someone-else", catalogueManifest(t, "avahi"), "t", nil, noBases); in.Kept != nil || !strings.Contains(in.None2, "avahi") {
|
||||
t.Errorf("a manifest of another module: kept %+v none2 %q", in.Kept, in.None2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAContextIsNotPinned(t *testing.T) { // 1450 (the rule 1444 keeps: context not pinned, as today's form)
|
||||
// build-agent's manifest from mesh-controller at f2d7bc94 has an image artifact built from a context in another repository.
|
||||
in := inputsOf("build-agent", fixture(t, "controller-f2d7bc94-build-agent"), "t", nil, noBases)
|
||||
if in.Kept == nil || in.None2 != None2Context {
|
||||
t.Errorf("kept %+v none2 %q, want none2 %q", in.Kept, in.None2, None2Context)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// moduleNameOf is the module a manifest is of: the name the manifest itself gives (read here only to key the manifests
|
||||
// value, so one digest always has one module; the inputs are read by the catalogue's parser, inputs.go), else the
|
||||
// module a build record names the directory the path of.
|
||||
func (ix *Indexer) moduleNameOf(repository, dir string, raw string) string {
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
}
|
||||
if json.Unmarshal([]byte(raw), &m) == nil && m.Module != "" {
|
||||
return m.Module
|
||||
}
|
||||
name, _ := ix.Builds.Module(repository, dir)
|
||||
return name
|
||||
}
|
||||
|
||||
// keepManifest writes manifests.<digest> for a manifest as the forge answered it, create-once (issue 1449): a key that
|
||||
// exists with the same value is left and is no error; one that holds another value is an error (it cannot happen for
|
||||
// one digest, ManifestDigest being a hash of the bytes, so it says the bucket was written by hand). It answers the
|
||||
// digest. A manifest that cannot be a value (empty, not UTF-8, not JSON) is an error the caller says; it is never tried
|
||||
// again. With no manifests store nothing is written and the digest is still answered.
|
||||
func (ix *Indexer) keepManifest(ctx context.Context, module, raw string) (string, error) {
|
||||
digest := delivery.ManifestDigest([]byte(raw))
|
||||
if ix.Manifests == nil {
|
||||
return digest, nil
|
||||
}
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: raw})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: manifests.%s: %v", errCannotBeAValue, key, err)
|
||||
}
|
||||
err = ix.Manifests.Create(ctx, key, data)
|
||||
if err == nil {
|
||||
return digest, nil
|
||||
}
|
||||
if !errors.Is(err, ErrConflict) {
|
||||
return "", fmt.Errorf("writing manifests.%s: %w", key, err)
|
||||
}
|
||||
held, _, found, err := ix.Manifests.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("reading manifests.%s: %w", key, err)
|
||||
}
|
||||
if !found {
|
||||
return "", fmt.Errorf("manifests.%s was there and is not: written again at the next delivery", key)
|
||||
}
|
||||
if cur, err := delivery.DecodeManifest(held); err == nil && delivery.SameManifest(cur, delivery.Manifest{V: delivery.Version, Module: module, JSON: raw}) {
|
||||
return digest, nil
|
||||
}
|
||||
return "", fmt.Errorf("manifests.%s holds another value than the manifest of %s, which hashes to it", key, module)
|
||||
}
|
||||
|
||||
// keepManifests writes the manifest of each directory a message carries, and answers the digest of each by directory.
|
||||
// A manifest that cannot be kept is left out and said in the log, so the module reads as one with no manifest.
|
||||
func (ix *Indexer) keepManifests(ctx context.Context, f delivery.Forge) (map[string]string, error) {
|
||||
digests := make(map[string]string, len(f.Manifests))
|
||||
for dir, raw := range f.Manifests {
|
||||
module := ix.moduleNameOf(f.Repository, dir, raw)
|
||||
digest, err := ix.keepManifest(ctx, module, raw)
|
||||
switch {
|
||||
case err == nil:
|
||||
digests[dir] = digest
|
||||
case errors.Is(err, errCannotBeAValue):
|
||||
ix.logf("%s#%d: the manifest of %q cannot be kept: %v", f.Repository, f.Number, dir, err)
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return digests, nil
|
||||
}
|
||||
|
||||
var errCannotBeAValue = errors.New("not a manifest value")
|
||||
@@ -0,0 +1,102 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The manifests these tests carry are real files, copied unchanged into testdata: the module.json of modules of
|
||||
// mesh-catalog at its main commit a67f01963f0d9ef0913a63720b89e573ca7a16ef (testdata/catalogue-a67f0196-<module>.module.json)
|
||||
// and of build-agent from mesh-controller at its main commit f2d7bc94 (testdata/controller-f2d7bc94-build-agent.module.json).
|
||||
func fixture(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile("testdata/" + name + ".module.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func catalogueManifest(t *testing.T, module string) string {
|
||||
return fixture(t, "catalogue-a67f0196-"+module)
|
||||
}
|
||||
|
||||
// avahiGoSum is the sha256 of modules/avahi/go.sum in mesh-catalog at a67f0196 (sha256sum of that file).
|
||||
const avahiGoSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||
|
||||
const catalogueRepo = "novox/mesh-catalog"
|
||||
|
||||
func manifestsIndexer(t *testing.T, recs ...[2]any) (*Indexer, *memStore, *memStore) {
|
||||
t.Helper()
|
||||
ix, mods := modulesIndexer(t, recs...)
|
||||
manifests := newMem()
|
||||
ix.Manifests = manifests
|
||||
return ix, mods, manifests
|
||||
}
|
||||
|
||||
func withManifests(f delivery.Forge, manifests map[string]string, locks map[string]map[string]string) delivery.Forge {
|
||||
f.Manifests, f.Locks = manifests, locks
|
||||
return f
|
||||
}
|
||||
|
||||
func TestAMergedManifestIsKeptByItsDigestAndNamedByTheModule(t *testing.T) { // 1449/AC1
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := delivery.ManifestDigest([]byte(avahi))
|
||||
raw, ok := manifests.vals[digest]
|
||||
if !ok || len(manifests.vals) != 1 {
|
||||
t.Fatalf("manifests holds %d keys, want only %s", len(manifests.vals), digest)
|
||||
}
|
||||
m, err := delivery.DecodeManifest(raw)
|
||||
if err != nil || m.Module != "avahi" || m.JSON != avahi {
|
||||
t.Errorf("manifest = %+v err %v, want avahi and the bytes of the file", m, err)
|
||||
}
|
||||
if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != digest {
|
||||
t.Errorf("module = %+v (%v), want the manifest digest %s", mod, ok, digest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSameManifestAppliedAgainIsWrittenOnce(t *testing.T) { // 1449/AC2
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, _, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A second merge, and a pull message carrying it, the digest being the same.
|
||||
f.Merge = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Errorf("a second merge: %v", err)
|
||||
}
|
||||
pull := delivery.Forge{V: delivery.Version, Kind: delivery.ForgePull, Repository: catalogueRepo, Branch: "feat/x", Number: 301, Head: h1,
|
||||
Action: delivery.PullUpdated, Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": avahi}}
|
||||
if err := ix.Apply(context.Background(), pull); err != nil {
|
||||
t.Errorf("a pull message: %v", err)
|
||||
}
|
||||
if manifests.writes != 1 || len(manifests.vals) != 1 {
|
||||
t.Errorf("manifests written %d times into %d keys, want once into one", manifests.writes, len(manifests.vals))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAManifestThatCannotBeAValueIsLeftOutAndSaid(t *testing.T) { // 1449 (a manifest that is not JSON cannot be kept byte for byte)
|
||||
var logs []string
|
||||
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
ix.Logf = func(format string, args ...any) { logs = append(logs, format) }
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": "not json"}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(manifests.vals) != 0 || len(logs) == 0 {
|
||||
t.Errorf("manifests %d keys, log %v", len(manifests.vals), logs)
|
||||
}
|
||||
if mod, ok := moduleOf(t, mods, "avahi"); !ok || mod.Manifest != "" {
|
||||
t.Errorf("module = %+v (%v), want no manifest digest", mod, ok)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package index
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -47,8 +48,14 @@ func fingerprintOf(k Known, trees map[string]string) (fp, none string) {
|
||||
// applyMerged writes, for each module built from the merged repository, its fingerprint at the merge commit (or none and
|
||||
// why), with the source revision that names the merge commit as the repository's head when the indexer has seen it
|
||||
// (ApplySource), and Src 0 when it has not. A module of another repository is not written. A value the module already
|
||||
// holds is not written again.
|
||||
// holds is not written again. From the manifests and lock hashes the message carries (the forge bridge, issue 1448) it
|
||||
// also writes the module's manifest digest, its new-form inputs and why it has no new-form fingerprint (issues 1444, 1449,
|
||||
// 1450); this release writes no fingerprint of the new form for any module.
|
||||
func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
digests, err := ix.keepManifests(ctx, f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, k := range ix.Builds.Modules(f.Repository) {
|
||||
key, err := delivery.ModulesKey(k.Module)
|
||||
if err != nil {
|
||||
@@ -56,10 +63,22 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
continue
|
||||
}
|
||||
fp, none := fingerprintOf(k, f.Trees)
|
||||
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge)}
|
||||
next := delivery.Module{Repository: f.Repository, Commit: f.Merge, FP: fp, None: none, Src: ix.srcOf(f.Repository, f.Merge), Manifest: digests[k.Path]}
|
||||
// The new form (issue 1444, R4c-2): no module has a fingerprint of it yet, so each has the reason it has none, which
|
||||
// names what is missing, from its manifest and lock hashes (inputs.go).
|
||||
raw, has := f.Manifests[k.Path]
|
||||
switch {
|
||||
case !has:
|
||||
next.None2 = None2NoManifest
|
||||
case next.Manifest == "":
|
||||
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
|
||||
default:
|
||||
in := inputsOf(k.Module, raw, f.Trees[k.Path], f.Locks[k.Path], ix.baseFP(ctx))
|
||||
next.Inputs2, next.None2 = in.Kept, in.None2
|
||||
}
|
||||
err = write(ctx, ix.Modules, key, func(value []byte, found bool) ([]byte, bool, error) {
|
||||
if found {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && delivery.SameModule(cur, withV(next)) {
|
||||
if cur, err := delivery.DecodeModule(value); err == nil && reflect.DeepEqual(cur, withV(next)) {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
@@ -73,4 +92,31 @@ func (ix *Indexer) applyMerged(ctx context.Context, f delivery.Forge) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// baseFP answers the new-form fingerprint of a base module as modules.<base> holds it, or why it has none.
|
||||
func (ix *Indexer) baseFP(ctx context.Context) baseFP {
|
||||
return func(module string) (string, string) {
|
||||
key, err := delivery.ModulesKey(module)
|
||||
if err != nil {
|
||||
return "", err.Error()
|
||||
}
|
||||
value, _, found, err := ix.Modules.Get(ctx, key)
|
||||
if err != nil {
|
||||
return "", "its record cannot be read: " + err.Error()
|
||||
}
|
||||
if !found {
|
||||
return "", "the indexer holds no record of it"
|
||||
}
|
||||
cur, err := delivery.DecodeModule(value)
|
||||
switch {
|
||||
case err != nil:
|
||||
return "", "its record cannot be read: " + err.Error()
|
||||
case cur.FP2 != "":
|
||||
return cur.FP2, ""
|
||||
case cur.None2 != "":
|
||||
return "", cur.None2
|
||||
}
|
||||
return "", "its record carries no fingerprint of the new form"
|
||||
}
|
||||
}
|
||||
|
||||
func withV(m delivery.Module) delivery.Module { m.V = delivery.Version; return m }
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package index
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// wantAvahiFP is today's fingerprint, at the tree "newtree", of a module with the recorded inputs aInputs (modules_test.go).
|
||||
func wantAvahiFP(t *testing.T) string {
|
||||
t.Helper()
|
||||
fp, none := delivery.SourceFingerprint(delivery.SourceInputs{Module: "a", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
|
||||
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
|
||||
if none != "" {
|
||||
t.Fatal(none)
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t *testing.T) { // 1444/AC1
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z",
|
||||
`{"Module":"avahi","Tree":"oldtree","Bases":["registry.example/base@sha256:aaa"],"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
|
||||
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "avahi")
|
||||
if !ok {
|
||||
t.Fatal("no module avahi")
|
||||
}
|
||||
todays, _ := delivery.SourceFingerprint(delivery.SourceInputs{Module: "avahi", Tree: "newtree", Bases: []string{"registry.example/base@sha256:aaa"},
|
||||
Toolchains: map[string]string{"code": "go sha256:ccc 0123456789abcdef"}})
|
||||
if m.Manifest != delivery.ManifestDigest([]byte(avahi)) || m.Inputs2 == nil || m.Inputs2.Tree != "newtree" || m.Inputs2.Locks["tools"] != avahiGoSum {
|
||||
t.Errorf("module = %+v, want its manifest digest and inputs", m)
|
||||
}
|
||||
if m.None2 != "toolchain not yet reported" || m.FP2 != "" {
|
||||
t.Errorf("none2 = %q fp2 = %q, want none2 'toolchain not yet reported'", m.None2, m.FP2)
|
||||
}
|
||||
if m.FP != todays || m.None != "" {
|
||||
t.Errorf("fp = %q none = %q, want today's fingerprint %s unchanged", m.FP, m.None, todays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) { // 1444/AC2
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "ca-trust", catalogueRepo, "modules/ca-trust", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/ca-trust": "newtree"}), map[string]string{"modules/ca-trust": catalogueManifest(t, "ca-trust")}, nil)
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.None2 != "builder's fingerprint not yet known" || m.Inputs2 == nil {
|
||||
t.Errorf("module = %+v (%v), want none2 naming the builder's fingerprint", m, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMergedMessageWithoutManifestsIsWrittenAsTodayWithItsReason(t *testing.T) { // 1444/AC3
|
||||
ix, mods, manifests := manifestsIndexer(t, rec(t, "build-1", "a", "novox/mesh-delivery", "modules/a", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
if err := ix.Apply(context.Background(), mergedMsg("novox/mesh-delivery", map[string]string{"modules/a": "newtree"})); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, ok := moduleOf(t, mods, "a")
|
||||
if !ok || m.FP != wantAvahiFP(t) || m.None2 != "the message carries no manifest" || m.Inputs2 != nil || m.Manifest != "" || m.FP2 != "" {
|
||||
t.Errorf("module = %+v (%v), want today's fingerprint, none2 'the message carries no manifest' and no inputs", m, ok)
|
||||
}
|
||||
if len(manifests.vals) != 0 {
|
||||
t.Errorf("manifests holds %d keys", len(manifests.vals))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSameMergeAppliedAgainWritesNothing(t *testing.T) { // 1444 (a value the module already holds is not written again, with inputs kept)
|
||||
avahi := catalogueManifest(t, "avahi")
|
||||
ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "avahi", catalogueRepo, "modules/avahi", "main", "", "2026-10-19T09:00:00Z", aInputs))
|
||||
f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "newtree"}), map[string]string{"modules/avahi": avahi},
|
||||
map[string]map[string]string{"modules/avahi": {"go.sum": avahiGoSum}})
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := ix.Apply(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mods.writes != 1 {
|
||||
t.Errorf("modules written %d times, want once", mods.writes)
|
||||
}
|
||||
}
|
||||
Vendored
+69
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"module": "audit-logger",
|
||||
"version": "1",
|
||||
"slug": "audit",
|
||||
"consumes": [
|
||||
"**"
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "code",
|
||||
"kind": "bundle",
|
||||
"language": "typescript",
|
||||
"entrypoints": [
|
||||
"index.js",
|
||||
"tools/index.js"
|
||||
],
|
||||
"loads": [
|
||||
"index.js",
|
||||
"tools/index.js"
|
||||
],
|
||||
"env": {
|
||||
"AUDIT_LOG": "${dir:trail}/audit.log",
|
||||
"AUDIT_SPOOL": "${dir:spool}"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "trail",
|
||||
"path": "${dir:trail}",
|
||||
"class": "valuable",
|
||||
"why": "the audit trail, written nowhere else"
|
||||
},
|
||||
{
|
||||
"id": "spool",
|
||||
"path": "${dir:spool}",
|
||||
"class": "valuable",
|
||||
"why": "events the trail could not take yet; after the bus gives one up, the only copy"
|
||||
}
|
||||
]
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "trail",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "spool",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"unknown": "program"
|
||||
}
|
||||
],
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
]
|
||||
}
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"module": "avahi",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"package-manager",
|
||||
"service-manager"
|
||||
],
|
||||
"tools": [
|
||||
"avahi_status",
|
||||
"avahi_browse",
|
||||
"avahi_resolve",
|
||||
"avahi_services"
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "package",
|
||||
"type": "package",
|
||||
"package": "avahi"
|
||||
},
|
||||
{
|
||||
"id": "daemon",
|
||||
"type": "service",
|
||||
"unit": "avahi-daemon.service",
|
||||
"state": "running",
|
||||
"boot": "enabled"
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/avahi-tools",
|
||||
"binary": "avahi-tools",
|
||||
"loads": [
|
||||
"avahi-tools"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Vendored
+66
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"module": "ca-trust",
|
||||
"version": "1",
|
||||
"slug": "catrust",
|
||||
"capabilities": [
|
||||
"service-manager"
|
||||
],
|
||||
"requires": [
|
||||
"internal-acme-ca"
|
||||
],
|
||||
"seats": [
|
||||
{
|
||||
"name": "the-mesh-trust-anchor",
|
||||
"scope": "node"
|
||||
}
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "the-mesh-trust-anchor",
|
||||
"scope": "node"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": ".",
|
||||
"unknown": "program"
|
||||
},
|
||||
{
|
||||
"id": "anchor",
|
||||
"type": "file",
|
||||
"path": "${dir:state}/anchor",
|
||||
"mode": "0755",
|
||||
"template": "files/anchor"
|
||||
},
|
||||
{
|
||||
"id": "unit",
|
||||
"type": "file",
|
||||
"path": "/etc/systemd/system/mesh-ca-trust.service",
|
||||
"mode": "0644",
|
||||
"template": "files/mesh-ca-trust.service"
|
||||
},
|
||||
{
|
||||
"id": "trust",
|
||||
"type": "service",
|
||||
"unit": "mesh-ca-trust.service",
|
||||
"state": "running",
|
||||
"boot": "enabled",
|
||||
"restart-on": [
|
||||
"anchor",
|
||||
"unit"
|
||||
]
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Vendored
+57
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"module": "netcheck",
|
||||
"version": "1",
|
||||
"emits": [
|
||||
"tunnel.endpoint-changed",
|
||||
"tunnel.handshake-stale",
|
||||
"tunnel.handshake-fresh"
|
||||
],
|
||||
"tools": [
|
||||
"netcheck_tunnel",
|
||||
"netcheck_tcp",
|
||||
"netcheck_dns",
|
||||
"netcheck_http",
|
||||
"netcheck_listening"
|
||||
],
|
||||
"replaces": {
|
||||
"netcheck_listening": [
|
||||
"ss -lunt",
|
||||
"ss -ltn",
|
||||
"netstat -lnt"
|
||||
],
|
||||
"netcheck_dns": [
|
||||
"dig",
|
||||
"nslookup"
|
||||
],
|
||||
"netcheck_tunnel": [
|
||||
"wg show",
|
||||
"wg show mesh0 dump"
|
||||
]
|
||||
},
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "tools-go",
|
||||
"kind": "bundle",
|
||||
"language": "go",
|
||||
"system": "arch",
|
||||
"from": "cmd/netcheck",
|
||||
"binary": "netcheck",
|
||||
"loads": [
|
||||
"netcheck"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "tools-typescript",
|
||||
"kind": "bundle",
|
||||
"language": "typescript",
|
||||
"entrypoints": [
|
||||
"tools/index.js"
|
||||
],
|
||||
"loads": [
|
||||
"tools/index.js"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Vendored
+140
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"module": "build-agent",
|
||||
"version": "1",
|
||||
"slug": "agent",
|
||||
"capabilities": [
|
||||
"container-runtime"
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"name": "node-build-agent",
|
||||
"scope": "node",
|
||||
"serves": ["current", "kill", "pause", "resume"]
|
||||
}
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.pulls",
|
||||
"mesh-delivery.pulls"
|
||||
],
|
||||
"requires": [
|
||||
"artifact-store",
|
||||
{
|
||||
"name": "npm-package-registry",
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"binds": {
|
||||
"npm-package-registry": "${dir:mesh-state}/package-registry.json"
|
||||
},
|
||||
"secrets": {
|
||||
"npm-package-registry": "${dir:mesh-state}/package-registry.secret"
|
||||
},
|
||||
"own-secrets": {
|
||||
"broker": "${dir:mesh-state}/broker",
|
||||
"genesis-package-registry": {
|
||||
"path": "${dir:mesh-state}/genesis-package-registry.json",
|
||||
"issued-by": "outside"
|
||||
}
|
||||
},
|
||||
"data": {
|
||||
"own": [
|
||||
{
|
||||
"id": "workspace",
|
||||
"path": "${dir:workspace}",
|
||||
"class": "cache",
|
||||
"why": "a build's working copy, cloned again for every build"
|
||||
},
|
||||
{
|
||||
"id": "judge-cache",
|
||||
"path": "${dir:judge-cache}",
|
||||
"class": "cache",
|
||||
"why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)"
|
||||
}
|
||||
]
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"id": "mesh-state",
|
||||
"type": "directory",
|
||||
"mode": "0700",
|
||||
"place": "mesh"
|
||||
},
|
||||
{
|
||||
"id": "workspace",
|
||||
"type": "directory",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "judge-cache",
|
||||
"type": "directory",
|
||||
"mode": "0700"
|
||||
},
|
||||
{
|
||||
"id": "agent-env",
|
||||
"type": "file",
|
||||
"path": "${dir:mesh-state}/build-agent.env",
|
||||
"mode": "0600",
|
||||
"template": "files/build-agent.env"
|
||||
},
|
||||
{
|
||||
"id": "server",
|
||||
"type": "container",
|
||||
"name": "mesh-build-agent",
|
||||
"artifact": "server",
|
||||
"env-file": [
|
||||
"${dir:mesh-state}/build-agent.env"
|
||||
],
|
||||
"volumes": [
|
||||
"${dir:mesh-state}:/run/mesh:ro",
|
||||
"${dir:workspace}:${dir:workspace}",
|
||||
"${dir:judge-cache}:${dir:judge-cache}",
|
||||
"/var/run/docker.sock:/var/run/docker.sock"
|
||||
],
|
||||
"restart-on": [
|
||||
"agent-env"
|
||||
],
|
||||
"network": "host",
|
||||
"health": {
|
||||
"kind": "exec",
|
||||
"command": "/usr/local/bin/mesh-builder ready",
|
||||
"interval": "10s",
|
||||
"timeout": "5s",
|
||||
"looks": 3,
|
||||
"grace": "60s"
|
||||
},
|
||||
"busy": {
|
||||
"command": "/usr/local/bin/mesh-builder busy"
|
||||
}
|
||||
}
|
||||
],
|
||||
"build": {
|
||||
"artifacts": [
|
||||
{
|
||||
"name": "server",
|
||||
"kind": "image",
|
||||
"from": "Dockerfile",
|
||||
"compiles": "cmd/mesh-builder",
|
||||
"context": {
|
||||
"seat": "git",
|
||||
"repository": "novox/mesh-controller",
|
||||
"ref": "main"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "files",
|
||||
"kind": "archive",
|
||||
"from": "files"
|
||||
}
|
||||
],
|
||||
"on": [
|
||||
{
|
||||
"arg": "GO_BASE",
|
||||
"image": "golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c"
|
||||
},
|
||||
{
|
||||
"arg": "ALPINE_BASE",
|
||||
"image": "alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -76,7 +76,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1
|
||||
if m.Module != "mesh-delivery-indexer" {
|
||||
t.Errorf("module = %q", m.Module)
|
||||
}
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
|
||||
if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) {
|
||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||
}
|
||||
if want := []string{"mesh-controller.builds", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
],
|
||||
"state": [
|
||||
{ "name": "candidates", "history": 10 },
|
||||
{ "name": "manifests", "history": 1 },
|
||||
{ "name": "modules", "history": 20 }
|
||||
],
|
||||
"reads": [
|
||||
|
||||
Reference in New Issue
Block a user