Take each mesh package's latest version into the fingerprint

ADR 0573 §2: a module builds against the latest published version of
each mesh package it uses, and a package's trunk merge moves every
dependent (issue 1585, S13). The bridge reads the packages each go.sum
names, the indexer keeps them and takes each one's trunk merge into the
fingerprint, a candidate takes a group member's head instead, and the
builder's ask carries the versions so the build module writes them.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
This commit is contained in:
2026-11-01 02:07:04 +01:00
parent b1f262d7d0
commit cb17d59745
21 changed files with 242 additions and 64 deletions
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0 h1:q6ywchpTM9fwaJ1UNwDSuOjb5FYDFcsaeiTrKoB4scc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -68,7 +68,7 @@ func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
continue
}
m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path,
Toolchains: b.agreedFor(ctx, cm.Manifest)}
Toolchains: b.agreedFor(ctx, cm.Manifest), Packages: cm.Packages} // the group's package heads (ADR 0573 §4)
if len(cm.Paths) > 0 {
// The named paths' objects the preview gave it: the fetch names the paths (runLane, ADR 0499 clause 4).
m.Inputs2 = &delivery.KeptInputs2{Paths: cm.Paths}
@@ -808,6 +808,11 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
job.Fetch = req.Args()
}
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: language, Asker: delivery.BuilderModule, At: time.Now(), Build: job}
if language == delivery.AskGo && job.Fetch != nil && len(m.Packages) > 0 {
// The versions of the mesh packages the fingerprint was computed with (novox/hq ADR 0573 §2): the build module writes
// them into the workspace, whatever the repository's go.mod names.
ask.Packages = maps.Clone(m.Packages)
}
if id := asked.resume(language); id != "" {
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
if b.Held != nil {
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"reflect"
"strconv"
"strings"
"sync"
@@ -300,7 +301,14 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
r := newRig(t, goBundle)
r.asks.Answer = nil // no node has pulled it yet
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
// Issue 1585 (ADR 0573 §2): the ask carries the package versions the fingerprint was computed with.
sdk := delivery.PackageVersions{"git.novox.be/novox/mesh-sdk/go": hex40('5')}
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20",
FP2: fp1, Manifest: mdigest('a'), Path: "modules/m", Toolchains: agreedToolchains, Packages: sdk})
if err != nil {
t.Fatal(err)
}
r.b.Module(context.Background(), "m", data)
var asks []delivery.StepAsk
for deadline := time.Now().Add(2 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) {
if time.Now().After(deadline) {
@@ -312,6 +320,9 @@ func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
if len(asks) != 1 || a.Kind != delivery.AskDelivery || a.Language != delivery.AskGo || a.Build == nil || a.Build.FP != fp1 {
t.Fatalf("asks = %+v, want one Go delivery ask of the fingerprint", asks)
}
if !reflect.DeepEqual(a.Packages, sdk) {
t.Errorf("the ask's packages = %v, want %v", a.Packages, sdk)
}
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding || v.Asks[delivery.AskGo] != a.ID {
t.Fatalf("the key = %+v, want building naming the ask %s", v, a.ID)
}
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2 h1:RF73nVI1dTrrmlFosVDTMJmpUQ+rEeB7++BHH/RvZFg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030201806-eff81f1220e2/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0 h1:q6ywchpTM9fwaJ1UNwDSuOjb5FYDFcsaeiTrKoB4scc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -151,7 +151,7 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep
if action == delivery.PullUpdated && p.ModuleDirsSaid {
f.Dirs, f.DirsKnown = p.ModuleDirs, true
var err error
if f.Manifests, f.Locks, err = inputsOf(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
if f.Manifests, f.Locks, f.Packages, _, err = inputsRead(p.Owner, p.Repo, p.HeadSHA, p.ModuleDirs, files); err != nil {
return Message{}, fmt.Errorf("%w: %s: the manifests of %s/%s at the head %s: %v", ErrRetry, subject, p.Owner, p.Repo, p.HeadSHA, err)
}
}
@@ -190,14 +190,15 @@ type read struct {
// order the calls finish in, and the error is that of the first failing file in the order of dirs and LockFiles. After
// a failure no further file is asked for, and inputsOf returns only when every call it started has returned.
func inputsOf(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, err error) {
manifests, locks, _, err = inputsRead(owner, repo, ref, dirs, files)
manifests, locks, _, _, err = inputsRead(owner, repo, ref, dirs, files)
return manifests, locks, err
}
// inputsRead is inputsOf, and also answers, on a failure too, the directories whose manifest and lock files were all
// read, with their manifests and lock hashes: a merge keeps them by their tree, so its next try reads only the rest
// (issue 1126).
func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, whole []string, err error) {
// inputsRead is inputsOf, and also answers the mesh packages each directory's go.sum names (delivery.MeshPackagesOf,
// novox/hq ADR 0573 §2; issue 1585) and, on a failure too, the directories whose manifest and lock files were all read,
// with their manifests and lock hashes: a merge keeps them by their tree, so its next try reads only the rest (issue
// 1126).
func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests map[string]string, locks map[string]map[string]string, packages map[string][]string, whole []string, err error) {
at := func(dir, name string) string {
if dir == "" {
return name
@@ -241,6 +242,7 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
wg.Wait()
manifests = map[string]string{}
locks = map[string]map[string]string{}
packages = map[string][]string{}
broken := map[string]bool{} // a directory with a file not read (failed, or never asked after a failure)
for _, r := range reads {
switch {
@@ -260,6 +262,11 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
locks[r.dir] = map[string]string{}
}
locks[r.dir][path.Base(r.path)] = hex.EncodeToString(sum[:])
if path.Base(r.path) == "go.sum" {
if p := delivery.MeshPackagesOf(r.raw); len(p) > 0 {
packages[r.dir] = p
}
}
}
}
for _, dir := range dirs {
@@ -271,8 +278,9 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
for dir := range broken {
delete(manifests, dir)
delete(locks, dir)
delete(packages, dir)
}
return manifests, locks, whole, err
return manifests, locks, packages, whole, err
}
if len(manifests) == 0 {
manifests = nil
@@ -280,7 +288,10 @@ func inputsRead(owner, repo, ref string, dirs []string, files Files) (manifests
if len(locks) == 0 {
locks = nil
}
return manifests, locks, whole, nil
if len(packages) == 0 {
packages = nil
}
return manifests, locks, packages, whole, nil
}
// merged is the merged message of a merge into the trunk: the trees of the merge commit, asked once.
@@ -301,15 +312,19 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
names = append(names, dir)
}
slices.Sort(names)
manifests, locks, toRead := keptInputs(names, dirs, kept)
read, readLocks, whole, err := inputsRead(p.Owner, p.Repo, p.MergeSHA, toRead, files)
manifests, locks, packages, toRead := keptInputs(names, dirs, kept)
read, readLocks, readPackages, whole, err := inputsRead(p.Owner, p.Repo, p.MergeSHA, toRead, files)
if err != nil {
keep(whole, dirs, read, readLocks, kept) // what was read whole is not asked again at the next try
keep(whole, dirs, read, readLocks, readPackages, kept) // what was read whole is not asked again at the next try
return Message{}, fmt.Errorf("%w: %s: the manifests of %s at the merge commit %s (%d of %d module directories read from the forge, the others kept by their tree): %v", ErrRetry, subject, repository, p.MergeSHA, len(toRead), len(names), err)
}
keep(toRead, dirs, read, readLocks, kept)
keep(toRead, dirs, read, readLocks, readPackages, kept)
maps.Copy(manifests, read)
maps.Copy(locks, readLocks)
maps.Copy(packages, readPackages)
if len(packages) == 0 {
packages = nil
}
if len(manifests) == 0 {
manifests = nil
}
@@ -321,7 +336,7 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag
return Message{}, unreadable(subject, "%v", err)
}
f := delivery.Forge{Kind: delivery.ForgeMerged, Repository: repository, Branch: p.Head, Number: p.Number,
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks}
Head: p.HeadSHA, Merge: p.MergeSHA, Root: root, Trees: dirs, Manifests: manifests, Locks: locks, Packages: packages}
if repository == CatalogRepository {
// The catalogue's predefined list at the merge (novox/hq ADR 0565 §2): the indexer keeps it for mesh-config.
raw, err := files(p.Owner, p.Repo, PredefinedFile, p.MergeSHA)
@@ -79,6 +79,10 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
if want := map[string]map[string]string{"modules/a": {"go.sum": hexSum(sum)}}; !reflect.DeepEqual(f.Locks, want) {
t.Errorf("locks = %v, want %v", f.Locks, want)
}
// Issue 1585 (ADR 0573 §2): and the mesh packages each go.sum names.
if want := map[string][]string{"modules/a": {"git.novox.be/novox/mesh-sdk/go"}}; !reflect.DeepEqual(f.Packages, want) {
t.Errorf("packages = %v, want %v", f.Packages, want)
}
// ADR 0567: placement is the seat mesh-placement's: a merge of the catalogue carries no placement file and asks the
// forge for none.
forge.calls = nil
@@ -124,14 +128,15 @@ func TestMergedCarriesManifestsAndLocks(t *testing.T) { // 1448/AC1
func TestPullWithKnownDirsCarriesTheSameAtTheHead(t *testing.T) { // 1448/AC2
manifest := real(t, "module.json")
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest}}
forge := &forgeFiles{have: map[string][]byte{"modules/a/module.json": manifest, "modules/a/go.sum": []byte("git.novox.be/novox/mesh-sdk/go v0.1.7 h1:x=\n")}}
msg, err := Convert(updatedSubject, updated(nil), nil, forge.get)
if err != nil {
t.Fatal(err)
}
f, _ := delivery.DecodeForge(msg.Data)
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 0 {
t.Errorf("manifests %v locks %v", f.Manifests, f.Locks)
if !reflect.DeepEqual(f.Manifests, map[string]string{"modules/a": string(manifest)}) || len(f.Locks) != 1 ||
!reflect.DeepEqual(f.Packages, map[string][]string{"modules/a": {"git.novox.be/novox/mesh-sdk/go"}}) {
t.Errorf("manifests %v locks %v packages %v", f.Manifests, f.Locks, f.Packages)
}
if len(forge.calls) == 0 || forge.calls[0].ref != head {
t.Errorf("calls %v, want the head %s", forge.calls, head)
@@ -389,4 +394,14 @@ func mergeReadsOnlyChangedDirs(t *testing.T) {
if len(failing.calls) != 3 {
t.Errorf("the try after a failure read %d files, want only modules/b's 3", len(failing.calls))
}
// Issue 1585: inputs kept before the bridge read the packages are read again, so no module loses the packages it uses.
for tree, in := range kept2 {
in.Packages, in.PackagesRead = nil, false
kept2[tree] = in
}
failing.calls = nil
again, err = ConvertKept(mergedSubject, mergedBody(nil), treesOf(&calls, "r1", trees, nil), failing.get, kept2)
if err != nil || string(again.Data) != string(whole.Data) || len(failing.calls) != 6 {
t.Errorf("inputs kept without their packages: %d files read, want 6; same message: %v (%v)", len(failing.calls), string(again.Data) == string(whole.Data), err)
}
}
@@ -7,11 +7,14 @@ package bridge
// every byte under it, so the manifest and the lock-file hashes read once at a tree hash are those of every later commit
// whose directory has the same tree: a merge reads the forge only for the directories it changed.
// Inputs is what the bridge read of one module directory at one tree hash: its manifest and its lock files' hex sha256
// by name (none when it has none).
// Inputs is what the bridge read of one module directory at one tree hash: its manifest, its lock files' hex sha256
// by name (none when it has none) and the mesh packages its go.sum names (novox/hq ADR 0573 §2). PackagesRead says the
// packages were read: a value kept before the bridge read them is not taken, since its packages are not known.
type Inputs struct {
Manifest string `json:"manifest"`
Locks map[string]string `json:"locks,omitempty"`
Manifest string `json:"manifest"`
Locks map[string]string `json:"locks,omitempty"`
Packages []string `json:"packages,omitempty"`
PackagesRead bool `json:"packages-read,omitempty"`
}
// Kept holds Inputs by a directory's git tree hash (the bridge's state bucket inputs). Get answers false for a tree it
@@ -22,8 +25,8 @@ type Kept interface {
}
// keptInputs splits dirs into the inputs kept for their tree hash and the directories still to read from the forge.
func keptInputs(dirs []string, trees map[string]string, kept Kept) (manifests map[string]string, locks map[string]map[string]string, read []string) {
manifests, locks = map[string]string{}, map[string]map[string]string{}
func keptInputs(dirs []string, trees map[string]string, kept Kept) (manifests map[string]string, locks map[string]map[string]string, packages map[string][]string, read []string) {
manifests, locks, packages = map[string]string{}, map[string]map[string]string{}, map[string][]string{}
for _, dir := range dirs {
tree := trees[dir]
if kept == nil || tree == "" {
@@ -31,7 +34,7 @@ func keptInputs(dirs []string, trees map[string]string, kept Kept) (manifests ma
continue
}
in, ok := kept.Get(tree)
if !ok {
if !ok || !in.PackagesRead {
read = append(read, dir)
continue
}
@@ -39,19 +42,22 @@ func keptInputs(dirs []string, trees map[string]string, kept Kept) (manifests ma
if len(in.Locks) > 0 {
locks[dir] = in.Locks
}
if len(in.Packages) > 0 {
packages[dir] = in.Packages
}
}
return manifests, locks, read
return manifests, locks, packages, read
}
// keep puts what was read of each directory under its tree hash.
func keep(dirs []string, trees map[string]string, manifests map[string]string, locks map[string]map[string]string, kept Kept) {
func keep(dirs []string, trees map[string]string, manifests map[string]string, locks map[string]map[string]string, packages map[string][]string, kept Kept) {
if kept == nil {
return
}
for _, dir := range dirs {
raw, ok := manifests[dir]
if tree := trees[dir]; ok && tree != "" {
kept.Put(tree, Inputs{Manifest: raw, Locks: locks[dir]})
kept.Put(tree, Inputs{Manifest: raw, Locks: locks[dir], Packages: packages[dir], PackagesRead: true})
}
}
}
@@ -165,6 +165,14 @@ func readVia(call toolCaller) func(ctx context.Context, repository, sha, dir str
out.Locks[dir] = map[string]string{}
}
out.Locks[dir][name] = hex.EncodeToString(sum[:])
if name == "go.sum" {
if p := delivery.MeshPackagesOf([]byte(c)); len(p) > 0 {
if out.Packages == nil {
out.Packages = map[string][]string{}
}
out.Packages[dir] = p // the mesh packages it uses, as the bridge reads them (ADR 0573 §2)
}
}
}
}
return out, nil
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
+2
View File
@@ -1,5 +1,7 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a h1:DcSvfEmdPBKqCg5OmXRNb269EkkPe7BsEp+RykEhkgE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261031084708-acbf80ad910a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0 h1:q6ywchpTM9fwaJ1UNwDSuOjb5FYDFcsaeiTrKoB4scc=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101005726-af4e4e60a3a0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
@@ -58,7 +58,7 @@ type baseFP func(module string) (fp, none string)
// here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve
// nothing and have no toolchain. A base is a build.on entry that names a module.
func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs {
return inputsOfNamed(module, raw, tree, locks, nil, base)
return inputsOfNamed(module, raw, tree, locks, nil, nil, base)
}
// inputsOfNamed is inputsOf with the git object of each path the message names for the module's directory (delivery
@@ -66,7 +66,10 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo
// manifest, by mesh-sdk's one function (delivery.NamedTree), and keeps their objects, so a computation from kept inputs
// gives the merge's value (novox/hq ADR 0499 clause 2); a message that does not carry the object of each such path gives it
// none, naming the paths (clause 3). Every other module keeps its directory's tree, whatever objects are given.
func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, base baseFP) moduleInputs {
//
// packages are the mesh packages of other repositories the module's go.sum names (novox/hq ADR 0573 §2): a module with a
// Go artifact keeps them, and the fingerprint takes the latest published version of each (Recompute).
func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, packages []string, base baseFP) moduleInputs {
m, err := catalogue.ParseManifestForInputs([]byte(raw))
if err != nil {
return moduleInputs{None2: None2Unreadable + err.Error()}
@@ -133,6 +136,10 @@ func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, b
}
}
}
if usesGo(langs) && len(packages) > 0 {
kept.Packages = slices.Clone(packages)
sort.Strings(kept.Packages)
}
sort.Strings(kept.Bases)
sort.Strings(missing)
out := moduleInputs{Kept: &kept, ArtifactPaths: artifactPaths}
@@ -152,6 +159,17 @@ func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, b
return out
}
// usesGo is whether an artifact is built with Go: only a Go build resolves a mesh package by the version the build asks
// (novox/hq ADR 0573 §2); a Node package moves its dependents as a base (ADR 0498 §5).
func usesGo(langs map[string]string) bool {
for _, l := range langs {
if l == "go" {
return true
}
}
return false
}
// languagesOf is the language of each artifact of the manifest that has a toolchain (a bundle or a package of go or
// typescript), by artifact name: the artifacts a build module's toolchain enters the fingerprint of.
func languagesOf(m catalogue.Manifest) map[string]string {
@@ -85,12 +85,13 @@ func (ix *Indexer) applyManifests(ctx context.Context, f delivery.Forge, drop bo
if next.Manifest == "" {
next.None2 = "the manifest cannot be read: it cannot be kept as a value"
} else {
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], ix.baseFP(ctx))
packages := delivery.OtherPackages(f.Packages[dir], f.Repository) // its own repository's come with its tree
in := inputsOfNamed(name, raw, f.Trees[dir], f.Locks[dir], objects[dir], packages, ix.baseFP(ctx))
next.Inputs2, next.None2, next.ArtifactPaths = in.Kept, in.None2, in.ArtifactPaths
if in.Kept == nil && strings.HasPrefix(in.None2, None2Unreadable) {
// The manifest cannot be read by this SDK; the inputs the merge carried are kept all the same, so a later SDK
// that reads it judges the module again at start with no merge (issue 1923).
next.Inputs2 = unreadInputs(f.Trees[dir], f.Locks[dir], f.Paths[dir])
next.Inputs2 = unreadInputs(f.Trees[dir], f.Locks[dir], f.Paths[dir], packages)
}
if in.Kept == nil && namedErr[dir] != "" {
next.None2 = in.None2 + ": the forge could not say them: " + namedErr[dir]
@@ -283,7 +284,7 @@ const None2Unreadable = "the manifest cannot be read: "
// hashes by file name (the artifacts that own them are not known without the manifest) and the objects of the named
// paths the message carried; nil when the message carried no tree. Judging the manifest again (rejudged) turns them into
// the inputs of the module (issue 1923).
func unreadInputs(tree string, locks, paths map[string]string) *delivery.KeptInputs2 {
func unreadInputs(tree string, locks, paths map[string]string, packages []string) *delivery.KeptInputs2 {
if tree == "" {
return nil
}
@@ -294,5 +295,6 @@ func unreadInputs(tree string, locks, paths map[string]string) *delivery.KeptInp
if len(paths) > 0 {
in.Paths = maps.Clone(paths)
}
in.Packages = slices.Clone(packages)
return in
}
@@ -63,7 +63,7 @@ func TestAMessageWithoutTheNamedPathsObjectsGivesTheModuleNoneNamingThem(t *test
}
func TestAModuleNamingNoPathsKeepsItsDirectorysTree(t *testing.T) { // I7, ADR 0499 clause 2: every other module is unchanged
in := inputsOfNamed("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, map[string]string{"x": "1111111111111111111111111111111111111111"}, noBases)
in := inputsOfNamed("avahi", catalogueManifest(t, "avahi"), "avahitree", map[string]string{"go.sum": avahiGoSum}, map[string]string{"x": "1111111111111111111111111111111111111111"}, nil, noBases)
if in.Kept == nil || in.Kept.Tree != "avahitree" || in.Kept.Paths != nil {
t.Errorf("inputs = %+v, want the directory's tree and no paths", in.Kept)
}
@@ -79,10 +79,10 @@ func TestACandidatesModuleTakesTheNamedPathsObjectsFromThePreview(t *testing.T)
named, _ := delivery.NamedTree(digest, sdkObjects)
with := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: named, Manifest: digest, Paths: sdkObjects}}}
without := delivery.Preview{Modules: map[string]delivery.PreviewModule{".": {Tree: "roottree", Manifest: digest}}}
if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", without); !strings.Contains(got.None2, "named paths") {
if got := ix.candidateModule(context.Background(), "mesh-sdk", ".", without, nil, nil); !strings.Contains(got.None2, "named paths") {
t.Errorf("without the objects: %+v, want none naming the named paths", got)
}
got := ix.candidateModule(context.Background(), "mesh-sdk", ".", with)
got := ix.candidateModule(context.Background(), "mesh-sdk", ".", with, nil, nil)
if strings.Contains(got.None2, "named paths") || got.Manifest != digest {
t.Errorf("with the objects: %+v, want them taken (none only for what else is missing)", got)
}
@@ -147,6 +147,14 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
rev, heads := ix.source.rev, ix.source.heads
ix.mu.Unlock()
changed, read := false, false
// A package one of the candidate's pull requests publishes is built at that pull request's head, every other at its
// latest published version (novox/hq ADR 0573 §4): a group's members build against the group's own package.
members := map[string]string{}
for _, p := range c.Pulls {
members[delivery.RepositoryName(p.Repository)] = p.Head.Head
}
var held map[string]delivery.Module
listed := false
for i := range c.Pulls {
p := &c.Pulls[i]
pv, found, stale := ix.previewOf(heads, p.Repository, p.Head.Head)
@@ -154,17 +162,20 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
continue
}
read = true
if !listed {
var err error
if held, err = ix.heldModules(ctx); err != nil {
ix.logf("the modules cannot be listed, so no touched module's directory or package version is known: %v", err)
}
listed = true
}
// A touched module's directory: the one the modules bucket holds of the repository, else the preview's directory
// whose manifest is the one the pull request gave the module (a module the pull request adds; ADR 0501 clause 6).
dirs := map[string]string{}
if held, err := ix.heldModules(ctx); err == nil {
for name, m := range held {
if delivery.SameRepository(m.Repository, p.Repository) && m.Path != "" {
dirs[name] = m.Path
}
for name, m := range held {
if delivery.SameRepository(m.Repository, p.Repository) && m.Path != "" {
dirs[name] = m.Path
}
} else {
ix.logf("the modules cannot be listed, so no touched module's directory is known: %v", err)
}
for _, m := range p.Modules {
if _, known := dirs[m]; known || p.Fps[m].Manifest == "" {
@@ -190,7 +201,7 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
next.None2 = NoneNoDirectory
break
}
next = ix.candidateModule(ctx, m, dir, pv)
next = ix.candidateModule(ctx, m, dir, pv, held, members)
}
if fps == nil {
fps = map[string]delivery.CandidateModule{}
@@ -214,9 +225,10 @@ func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool
}
// candidateModule is the fingerprint of module m, whose directory is dir, at the preview: the module's inputs from the
// merge (tree, manifest, locks), its bases' and the builder's fingerprints as the trunk's modules hold them, and the toolchains
// the build modules agree on, by the rules Recompute applies to a module of the trunk.
func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delivery.Preview) delivery.CandidateModule {
// merge (tree, manifest, locks), its bases' and the builder's fingerprints as the trunk's modules hold them, the toolchains
// the build modules agree on, and the version of each mesh package it uses (candidatePackages), by the rules Recompute
// applies to a module of the trunk.
func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delivery.Preview, held map[string]delivery.Module, members map[string]string) delivery.CandidateModule {
pm, ok := pv.Modules[dir]
if !ok && dir == "" {
pm, ok = pv.Modules[delivery.RootPath]
@@ -242,7 +254,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
out.None2 = NonePreviewNoManifest
return out
}
in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, ix.baseFP(ctx))
in := inputsOfNamed(m, kept.JSON, pm.Tree, pm.Locks, pm.Paths, delivery.OtherPackages(pm.Packages, pv.Repository), ix.baseFP(ctx))
if in.Kept == nil {
out.None2 = in.None2
return out
@@ -262,14 +274,32 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive
toolchains[artifact] = t.tc
}
inputs := *in.Kept // no builder input (ADR 0530 §1)
fp, none := delivery.SourceFingerprint2(inputs.With(m, toolchains))
versions := candidatePackages(inputs.Packages, held, members)
fp, none := delivery.SourceFingerprint2(inputs.WithPackages(m, toolchains, versions))
if fp == "" {
out.None2 = none
return out
}
out.FP2 = fp
out.FP2, out.Packages = fp, versions
if len(inputs.Paths) > 0 {
out.Paths = inputs.Paths // the builder's fetch for the candidate names them (ADR 0499 clause 4)
}
return out
}
// candidatePackages is the version each package of a candidate's module is built against (novox/hq ADR 0573 §2, §4): the
// head of the candidate's pull request whose repository publishes it (members, by repository), else its latest published
// version, as the trunk's modules hold it. A package with neither is left out, and the fingerprint is then none.
func candidatePackages(packages []string, held map[string]delivery.Module, members map[string]string) delivery.PackageVersions {
versions, _ := delivery.PackageVersionsOf(packages, held)
for _, p := range packages {
repository, _, _ := delivery.PackageRepository(p)
if head := members[repository]; head != "" {
if versions == nil {
versions = delivery.PackageVersions{}
}
versions[p] = head
}
}
return versions
}
@@ -109,6 +109,24 @@ func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *test
if other := fpOf(t, cands); other.FP2 == "" || other.FP2 == trunk.FP2 {
t.Errorf("a changed tree: %+v", other)
}
// Issue 1585 (ADR 0573 §2, §4): a candidate's module is built against the head of the candidate's pull request that
// publishes a package it uses, else against the package's latest published version; a preview naming a package of
// mesh-sdk that nothing published gives no fingerprint, and one of its own repository's comes with its tree.
const sdk, own = "git.novox.be/novox/mesh-sdk/go", "git.novox.be/novox/mesh-catalog/contract"
held := map[string]delivery.Module{"mesh-sdk": {Repository: "novox/mesh-sdk", Path: delivery.RootPath, Commit: trunkOld}}
if got := candidatePackages([]string{sdk}, held, map[string]string{catalogueRepo: h1}); got[sdk] != trunkOld {
t.Errorf("outside a group: %v, want mesh-sdk at its trunk %s", got, trunkOld)
}
if got := candidatePackages([]string{sdk}, held, map[string]string{catalogueRepo: h1, "novox/mesh-sdk": mergeSha}); got[sdk] != mergeSha {
t.Errorf("in a group with mesh-sdk: %v, want its member's head %s", got, mergeSha)
}
m.Packages = []string{own, sdk}
pv.Modules["modules/avahi"] = m
pv.Merge = "8888888888888888888888888888888888888888"
applyPreview(t, ix, pv)
if got := fpOf(t, cands); got.FP2 != "" || got.None2 != delivery.NoPackageVersion+sdk {
t.Errorf("a package nothing published: %+v", got)
}
}
func TestAnUnknownToolchainLeavesTheCandidatesModuleNamingIt(t *testing.T) { // 1445/AC2
@@ -24,6 +24,8 @@ type RepoRead struct {
Trees map[string]string
Manifests map[string]string
Locks map[string]map[string]string
// Packages are the mesh packages each directory's go.sum names (delivery.MeshPackagesOf, novox/hq ADR 0573 §2).
Packages map[string][]string
}
// IndexRegistered writes the module of a registration as a merge of its repository would, when the modules bucket holds
@@ -129,6 +131,9 @@ func (ix *Indexer) indexAt(ctx context.Context, r Registration, force, reread bo
if l := read.Locks[dir]; len(l) > 0 {
f.Locks = map[string]map[string]string{dir: l}
}
if p := read.Packages[dir]; len(p) > 0 {
f.Packages = map[string][]string{dir: p}
}
if _, err := ix.applyManifests(ctx, f, false); err != nil {
return t, err
}
@@ -200,6 +200,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
type result struct {
fp, none string
tcs map[string]delivery.ToolchainUsed // by language, when there is a fingerprint
pkgs delivery.PackageVersions // the package versions, when there is a fingerprint
}
done := map[string]result{}
visiting := map[string]bool{}
@@ -251,6 +252,9 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
}
toolchains[artifact] = t.tc
}
// The latest published version of each mesh package the module uses: the trunk merge of the module that holds it,
// as this recompute holds it, so a package's merge moves every dependent in the same recompute (ADR 0573 §2).
versions, _ := delivery.PackageVersionsOf(kept.Packages, held)
var r result
switch {
case k.contexts:
@@ -259,13 +263,13 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
r.none = strings.Join(missing, "; ")
case noToolchain != "":
r.none = noToolchain
if fp, ok := stillComputed(module, kept, k, toolchains); ok {
r = result{fp: fp, tcs: k.held.Toolchains}
if fp, ok := stillComputed(module, kept, k, toolchains, versions); ok {
r = result{fp: fp, tcs: k.held.Toolchains, pkgs: versions}
}
default:
r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains))
r.fp, r.none = delivery.SourceFingerprint2(kept.WithPackages(module, toolchains, versions))
if r.fp != "" {
r.tcs = toolchainsUsed(toolchains)
r.tcs, r.pkgs = toolchainsUsed(toolchains), versions
}
}
done[module] = r
@@ -278,7 +282,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
r := of(key, nil)
if _, merged := overlay[key]; merged {
final := overlay[key]
final.FP2, final.None2, final.Inputs2, final.Toolchains = r.fp, r.none, k.cur.Inputs2, r.tcs
final.FP2, final.None2, final.Inputs2, final.Toolchains, final.Packages = r.fp, r.none, k.cur.Inputs2, r.tcs, r.pkgs
// What the bases gave is kept as it was while the value it gave does not change, so a merge
// that moves nothing for the module writes nothing (as Recompute does).
if value, _, found, err := ix.Modules.Get(ctx, key); err == nil && found {
@@ -291,7 +295,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
finals[key] = final
continue
}
if !stale[key] && k.cur.FP2 == r.fp && k.cur.None2 == r.none && reflect.DeepEqual(k.cur.Toolchains, r.tcs) {
if !stale[key] && k.cur.FP2 == r.fp && k.cur.None2 == r.none && reflect.DeepEqual(k.cur.Toolchains, r.tcs) && reflect.DeepEqual(k.cur.Packages, r.pkgs) {
continue
}
digest, inputs, paths := k.cur.Manifest, k.cur.Inputs2, k.cur.ArtifactPaths
@@ -304,10 +308,10 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli
if err != nil || cur.Manifest != digest || cur.Inputs2 == nil {
return nil, false, nil // a merge changed it meanwhile; its own recompute follows
}
if cur.FP2 == r.fp && cur.None2 == r.none && reflect.DeepEqual(cur.Toolchains, r.tcs) {
if cur.FP2 == r.fp && cur.None2 == r.none && reflect.DeepEqual(cur.Toolchains, r.tcs) && reflect.DeepEqual(cur.Packages, r.pkgs) {
return nil, false, nil
}
cur.FP2, cur.None2, cur.Toolchains = r.fp, r.none, r.tcs
cur.FP2, cur.None2, cur.Toolchains, cur.Packages = r.fp, r.none, r.tcs, r.pkgs
cur.Inputs2, cur.ArtifactPaths = inputs, paths
data, err := delivery.EncodeModule(cur)
wrote = err == nil
@@ -397,7 +401,7 @@ func (ix *Indexer) rejudged(ctx context.Context, m catalogue.Manifest, cur deliv
if cur.FP2 != "" || cur.Inputs2 == nil || !strings.HasPrefix(cur.None2, None2Unreadable) {
return cur, false
}
in := inputsOfNamed(m.Module, raw, cur.Inputs2.Tree, cur.Inputs2.Locks, cur.Inputs2.Paths, ix.baseFP(ctx))
in := inputsOfNamed(m.Module, raw, cur.Inputs2.Tree, cur.Inputs2.Locks, cur.Inputs2.Paths, cur.Inputs2.Packages, ix.baseFP(ctx))
if in.Kept == nil {
return cur, false
}
@@ -424,7 +428,7 @@ func hasContext(m catalogue.Manifest) bool {
// computed and built). Only computing a new fingerprint needs the agreement: the held one is kept when its kept inputs,
// with the toolchains it was computed with in place of the languages agreed on none, give it again. Any other input that
// moved (a merge, a base, an agreed toolchain) gives another fingerprint, and the module has none until they agree.
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2) (string, bool) {
func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed map[string]delivery.Toolchain2, versions delivery.PackageVersions) (string, bool) {
if k.held.FP2 == "" {
return "", false
}
@@ -440,7 +444,7 @@ func stillComputed(module string, kept delivery.KeptInputs2, k *known, agreed ma
}
toolchains[artifact] = delivery.Toolchain2{Language: language, Version: used.Version, Recipe: used.Recipe}
}
fp, none := delivery.SourceFingerprint2(kept.With(module, toolchains))
fp, none := delivery.SourceFingerprint2(kept.WithPackages(module, toolchains, versions))
return fp, none == "" && fp == k.held.FP2
}
@@ -294,6 +294,41 @@ func TestAModuleKeepsTheToolchainsItsFingerprintWasComputedWith(t *testing.T) {
if n.FP2 == m.FP2 || n.Toolchains["go"].Version != "go1.27.2" {
t.Errorf("after the upgrade: fp2 %q (was %q) toolchains %+v, want the new version with the new fingerprint", n.FP2, m.FP2, n.Toolchains)
}
// Issue 1585 (ADR 0573 §2): a module whose go.sum names mesh-sdk takes mesh-sdk's latest published version into its
// fingerprint and keeps it beside its toolchains; a mesh-sdk trunk merge moves it with no merge of its own, and a module
// using no package is not moved. With no published version held, it has none, naming the package.
const sdk = "git.novox.be/novox/mesh-sdk/go"
_, _, uses := newFormIndexerUnmerged(t, builderModules(t)...)
uses.Packages = map[string][]string{"modules/avahi": {sdk}}
if err := ix.Apply(context.Background(), uses); err != nil {
t.Fatal(err)
}
if u, _ := moduleOf(t, mods, "avahi"); u.FP2 != "" || u.None2 != delivery.NoPackageVersion+sdk {
t.Errorf("no published mesh-sdk: fp2 %q none2 %q", u.FP2, u.None2)
}
tools, _ := moduleOf(t, mods, "mesh-tools")
sdkMerge := func(commit, tree string) delivery.Module {
t.Helper()
f := delivery.Forge{V: delivery.Version, Kind: delivery.ForgeMerged, Repository: "novox/mesh-sdk", Branch: "feat/x", Number: 3, Head: h1,
Merge: commit, Root: tree, Trees: map[string]string{"": tree}, Manifests: map[string]string{"": `{"module":"mesh-sdk","version":"1"}`}}
if err := ix.Apply(context.Background(), f); err != nil {
t.Fatal(err)
}
a, _ := moduleOf(t, mods, "avahi")
if a.FP2 == "" || !reflect.DeepEqual(a.Packages, delivery.PackageVersions{sdk: commit}) {
t.Fatalf("after mesh-sdk's merge %.8s: fp2 %q none2 %q packages %v", commit, a.FP2, a.None2, a.Packages)
}
if o, _ := moduleOf(t, mods, "mesh-tools"); o.FP2 != tools.FP2 || o.Packages != nil {
t.Errorf("mesh-tools uses no package and moved: fp2 %q (was %q) packages %v", o.FP2, tools.FP2, o.Packages)
}
return a
}
first := sdkMerge(strings.Repeat("1", 40), "tree-sdk-1")
second := sdkMerge(strings.Repeat("2", 40), "tree-sdk-2")
if second.FP2 == first.FP2 {
t.Errorf("a mesh-sdk trunk merge kept avahi's fingerprint %q", first.FP2)
}
n = second
// Nodes that disagree keep the fingerprint already computed, with its toolchains (issue 1940, 2026-10-25 20:56: a
// stale report blanked every module and the controller's take-in refused modules already built); only a new
// fingerprint needs the agreement: a merge that moves the module's tree gives it none, naming the disagreement.