Merge pull request 'A module moves only when its program changes (hq issue 721)' (#336) from fix/a-module-moves-only-when-its-program-changes into main

This commit was merged in pull request #336.
This commit is contained in:
2026-10-14 20:21:05 +00:00
13 changed files with 811 additions and 4 deletions
+21
View File
@@ -36,6 +36,10 @@ type CheckBuilding struct {
Publish Publisher
Scratch Scratch
Held map[string]string
// Running is each artifact of the build the mesh has registered for its module, keyed "<module>/<artifact>"
// (novox/hq issue 721): what a build for the merge is compared with — never Held's newest build, which may be one
// its gate failed and the mesh put back.
Running map[string]string
Npmrc Npmrc
Seats map[string]string
// Named are the builds the build seat's verdicts of earlier checks of this pull request at this head name
@@ -57,6 +61,11 @@ type CheckBuilt struct {
// Kept is the check whose build this check kept instead of building, its key unchanged (novox/hq issue 703):
// the verdict names that build, by key and digest, as that check's. Empty when this check built it.
Kept string
// Same says every artifact came out the same as the build the mesh holds of the module (novox/hq issue 721), and
// SameAs names, per artifact, the held reference it was compared with: the walk leaves the module where it is
// while the mesh still holds that build. Empty when it differs, or could not be compared.
Same string
SameAs map[string]string
}
// skipsTheBuilds is why a check builds nothing for its merge; empty when it builds.
@@ -171,6 +180,18 @@ func makeForTheMerge(ctx context.Context, run Runner, spec CheckSpec, with Check
case result.ReuseKey == "":
built.NotBuilt = "it has no reuse key: " + result.NoKey
}
if built.NotBuilt == "" {
// **Compared with what runs** (novox/hq issue 721): a failed comparison says nothing, and the
// module moves as before.
same, against, err := sameAsHeld(ctx, run, dir, b.Module, result.Built, with.Running)
switch {
case err != nil:
say("build", "%s could not be compared with the build the mesh holds, so it moves: %v", b.Module, err)
case same != "":
built.Same, built.SameAs = same, against
say("build", "%s %s", b.Module, same)
}
}
out[i], results[i] = built, result
}(i, b, repository, at, filepath.Join(place, name))
}
+180
View File
@@ -0,0 +1,180 @@
package builder
import (
"archive/tar"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/catalogue"
)
// A module moves only when its program changes (novox/hq issue 721). A merge check builds every module the delivery
// plan moves; a module whose every artifact comes out the same as the build the mesh holds of it — the same files,
// the same modes and owners, the same entrypoint and environment — is said unchanged, and the walk leaves it where it
// is. **The whole image is compared, never its main binary alone**: a module whose image differs in any file moves.
// What is ignored is only what a rebuild cannot help changing: each file's time, the image's labels and its creation
// time. **A comparison that cannot be made says nothing**, and the module is built and moved as before.
// sameAsHeld says whether every artifact made is the same as the build the mesh holds of the module: the sentence
// that says so and, per artifact, the held reference it was compared with. Empty when any artifact differs, is not
// an image, has no held build, or could not be compared.
func sameAsHeld(ctx context.Context, run Runner, dir, module string, made []catalogue.Built,
held map[string]string) (string, map[string]string, error) {
if len(made) == 0 {
return "", nil, nil
}
against := map[string]string{}
for _, m := range made {
if m.Kind != "image" || m.Reference == "" {
return "", nil, nil
}
was := held[module+"/"+m.Name]
if was == "" {
return "", nil, nil
}
if digestPart(was) != "" && digestPart(was) == digestPart(m.Reference) {
against[m.Name] = was
continue
}
now, err := imageContent(ctx, run, dir, m.Reference)
if err != nil {
return "", nil, fmt.Errorf("%s's new image: %w", m.Name, err)
}
before, err := imageContent(ctx, run, dir, was)
if err != nil {
return "", nil, fmt.Errorf("%s's held image: %w", m.Name, err)
}
if now != before {
return "", nil, nil
}
against[m.Name] = was
}
var names []string
for name, ref := range against {
names = append(names, name+" "+shortRef(ref))
}
sort.Strings(names)
return "unchanged: its program is the same as running build " + strings.Join(names, ", "), against, nil
}
// digestPart is a reference's "sha256:…", empty when it is not pinned.
func digestPart(ref string) string {
if i := strings.LastIndex(ref, "@"); i >= 0 {
return ref[i+1:]
}
return ""
}
// shortRef names a reference by its digest's first twelve characters when it has one.
func shortRef(ref string) string {
if d := strings.TrimPrefix(digestPart(ref), "sha256:"); d != "" {
return twelve(d)
}
return ref
}
// imageContent is an image's content as one digest: its platform, its configuration without labels, and every file
// of its filesystem by path, type, mode, owner, link, extended attributes and bytes — never by time.
func imageContent(ctx context.Context, run Runner, dir, ref string) (string, error) {
if _, err := run(ctx, dir, "docker", "pull", "--quiet", ref); err != nil {
return "", fmt.Errorf("pulling %s: %w", ref, err)
}
config, err := run(ctx, dir, "docker", "image", "inspect", "--format",
"{{.Os}}/{{.Architecture}}/{{.Variant}} {{json .Config}}", ref)
if err != nil {
return "", fmt.Errorf("reading %s's configuration: %w", ref, err)
}
said, err := configContent(config)
if err != nil {
return "", fmt.Errorf("reading %s's configuration: %w", ref, err)
}
id, err := run(ctx, dir, "docker", "create", "--entrypoint", "/", ref)
if err != nil {
return "", fmt.Errorf("opening %s's filesystem: %w", ref, err)
}
id = strings.TrimSpace(id)
if id == "" {
return "", errors.New("the container engine named no container for " + ref)
}
defer func() { _, _ = run(context.WithoutCancel(ctx), dir, "docker", "rm", "--force", id) }()
if err := os.MkdirAll(dir, 0o700); err != nil {
return "", err
}
file := filepath.Join(dir, "content-"+twelve(id)+".tar")
defer os.Remove(file)
if _, err := run(ctx, dir, "docker", "export", "--output", file, id); err != nil {
return "", fmt.Errorf("reading %s's filesystem: %w", ref, err)
}
f, err := os.Open(file)
if err != nil {
return "", err
}
defer f.Close()
files, err := treeContent(f)
if err != nil {
return "", fmt.Errorf("reading %s's filesystem: %w", ref, err)
}
sum := sha256.Sum256([]byte(said + "\n" + files))
return "sha256:" + hex.EncodeToString(sum[:]), nil
}
// configContent is an image's platform and configuration as compared: Labels, the parent's id and the hostname the
// builder ran under are left out — the first holds the commit, the others no part of what runs.
func configContent(inspected string) (string, error) {
platform, raw, ok := strings.Cut(strings.TrimSpace(inspected), " ")
if !ok {
return "", fmt.Errorf("unexpected inspection %q", inspected)
}
var config map[string]json.RawMessage
if err := json.Unmarshal([]byte(raw), &config); err != nil {
return "", err
}
for _, k := range []string{"Labels", "Image", "Hostname", "Domainname"} {
delete(config, k)
}
out, err := json.Marshal(config) // map keys are sorted
if err != nil {
return "", err
}
return platform + " " + string(out), nil
}
// treeContent is a filesystem tarball's entries, sorted by path, each with everything but its times.
func treeContent(r io.Reader) (string, error) {
tr := tar.NewReader(r)
var lines []string
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return "", err
}
sum := sha256.New()
if _, err := io.Copy(sum, tr); err != nil {
return "", err
}
var xattrs []string
for k, v := range h.PAXRecords {
if strings.HasPrefix(k, "SCHILY.xattr.") {
xattrs = append(xattrs, k+"="+hex.EncodeToString([]byte(v)))
}
}
sort.Strings(xattrs)
lines = append(lines, fmt.Sprintf("%q %c %o %d:%d %q %d,%d %d %x %s",
strings.TrimPrefix(h.Name, "./"), h.Typeflag, h.Mode, h.Uid, h.Gid, h.Linkname, h.Devmajor, h.Devminor,
h.Size, sum.Sum(nil), strings.Join(xattrs, ";")))
}
sort.Strings(lines)
return strings.Join(lines, "\n"), nil
}
+126
View File
@@ -0,0 +1,126 @@
package builder
import (
"archive/tar"
"bytes"
"context"
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
// aFilesystem is a filesystem tarball of files, each written at the time given.
func aFilesystem(t *testing.T, at time.Time, files map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for _, name := range []string{"usr/local/bin/mesh-builder", "etc/apk/world"} {
body, ok := files[name]
if !ok {
continue
}
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o755, Size: int64(len(body)), ModTime: at,
Typeflag: tar.TypeReg}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
// anEngine answers the container engine's commands for images by reference: each image's filesystem and labels.
func anEngine(images map[string][]byte, labels map[string]string, failing string) Runner {
created := map[string]string{}
return func(_ context.Context, _ string, name string, args ...string) (string, error) {
joined := strings.Join(args, " ")
ref := args[len(args)-1]
if failing != "" && strings.Contains(joined, failing) {
return "", fmt.Errorf("the engine refused %s", joined)
}
switch {
case args[0] == "pull", args[0] == "rm":
return "", nil
case args[0] == "image":
return fmt.Sprintf(`linux/amd64/ {"Entrypoint":["/usr/local/bin/mesh-builder"],"Labels":{"commit":%q}}`,
labels[ref]), nil
case args[0] == "create":
id := fmt.Sprintf("container%04d", len(created))
created[id] = ref
return id + "\n", nil
case args[0] == "export":
return "", os.WriteFile(args[2], images[created[ref]], 0o600)
}
return "", fmt.Errorf("unexpected %s %s", name, joined)
}
}
// TestAModuleWhoseProgramDidNotChangeIsSaidUnchanged is novox/hq issue 721: mesh-controller #335 changed one sentence
// in a package mesh-builder imports and never reaches, and build-agent moved. Built again, its image differs only in
// the files' times and its commit label, and the check says it is the same as the build the mesh holds.
func TestAModuleWhoseProgramDidNotChangeIsSaidUnchanged(t *testing.T) {
held := map[string]string{"build-agent/agent": "store/build-agent@sha256:aaaa"}
made := []catalogue.Built{{Name: "agent", Kind: "image", Reference: "scratch/build-agent@sha256:bbbb"}}
files := map[string]string{"usr/local/bin/mesh-builder": "the same program", "etc/apk/world": "docker-cli\ngit\n"}
images := map[string][]byte{
"store/build-agent@sha256:aaaa": aFilesystem(t, time.Unix(1000, 0), files),
"scratch/build-agent@sha256:bbbb": aFilesystem(t, time.Unix(2000, 0), files),
}
labels := map[string]string{"store/build-agent@sha256:aaaa": "a5500f73", "scratch/build-agent@sha256:bbbb": "c2ec0a60"}
same, against, err := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent",
made, held)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(same, "unchanged: its program is the same as running build agent") ||
against["agent"] != held["build-agent/agent"] {
t.Fatalf("an image the same but for its times and labels is not said unchanged: %q %v", same, against)
}
// **Any file differing moves it**, not only its program.
changed := map[string]string{"usr/local/bin/mesh-builder": "the same program", "etc/apk/world": "docker-cli\ngit\ncurl\n"}
images["scratch/build-agent@sha256:bbbb"] = aFilesystem(t, time.Unix(2000, 0), changed)
if same, _, err := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent", made,
held); err != nil || same != "" {
t.Fatalf("an image with another file is said unchanged: %q %v", same, err)
}
// **A comparison that fails says nothing**, and the module moves.
images["scratch/build-agent@sha256:bbbb"] = aFilesystem(t, time.Unix(2000, 0), files)
if same, _, err := sameAsHeld(context.Background(), anEngine(images, labels, "export"), t.TempDir(), "build-agent",
made, held); err == nil || same != "" {
t.Fatalf("a comparison that failed said %q, %v", same, err)
}
// No held build, or an artifact that is not an image: nothing compared.
if same, _, _ := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent", made,
nil); same != "" {
t.Fatalf("a module the mesh holds no build of is said unchanged: %q", same)
}
archive := []catalogue.Built{{Name: "agent", Kind: "archive", Reference: "scratch/build-agent@sha256:bbbb"}}
if same, _, _ := sameAsHeld(context.Background(), anEngine(images, labels, ""), t.TempDir(), "build-agent",
archive, held); same != "" {
t.Fatalf("an archive is said unchanged without being compared: %q", same)
}
}
// TestAnImageWithAnotherEntrypointIsNotTheSame: the configuration counts, but for its labels.
func TestAnImageWithAnotherEntrypointIsNotTheSame(t *testing.T) {
a, err := configContent(`linux/amd64/ {"Entrypoint":["/a"],"Labels":{"commit":"1"}}`)
if err != nil {
t.Fatal(err)
}
b, _ := configContent(`linux/amd64/ {"Labels":{"commit":"2"},"Entrypoint":["/a"]}`)
c, _ := configContent(`linux/amd64/ {"Entrypoint":["/b"],"Labels":{"commit":"1"}}`)
if a != b || a == c {
t.Fatalf("labels count, or the entrypoint does not: %q %q %q", a, b, c)
}
}
+46
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"sort"
"strings"
"time"
"github.com/jackc/pgx/v5"
@@ -301,6 +302,51 @@ func (i *Inventory) Held(ctx context.Context) (map[string]string, error) {
return held, nil
}
// RegisteredHeld is Held narrowed to the build each module has registered (novox/hq issue 721): an artifact is kept
// only when its reference's digest is in the manifest the module table holds for it, which is what its machines are
// sent. After a gate failed and the mesh put the earlier build back (RestoreModule), the newest successful build is
// not what runs, and its artifacts are left out here, so nothing is compared with them.
func (i *Inventory) RegisteredHeld(ctx context.Context) (map[string]string, error) {
held, err := i.Held(ctx)
if err != nil {
return nil, err
}
rows, err := i.store.Pool().Query(ctx, `select name, coalesce(manifest::text, '') from module`)
if err != nil {
return nil, err
}
defer rows.Close()
registered := map[string]string{}
for rows.Next() {
var name, manifest string
if err := rows.Scan(&name, &manifest); err != nil {
return nil, err
}
registered[name] = manifest
}
if err := rows.Err(); err != nil {
return nil, err
}
return registeredOnly(held, registered), nil
}
// registeredOnly keeps of held each "<module>/<artifact>" whose reference's digest the module's registered manifest
// names.
func registeredOnly(held, registered map[string]string) map[string]string {
out := map[string]string{}
for key, ref := range held {
module, _, ok := strings.Cut(key, "/")
at := strings.LastIndex(ref, "@")
if !ok || at < 0 || len(ref) <= at+1 {
continue
}
if strings.Contains(registered[module], ref[at+1:]) {
out[key] = ref
}
}
return out
}
// BuiltAgainst is what each module's newest successful build stood on, as recorded — the build
// edges (ADR 0009). A module whose last build recorded no bases is absent, which is also what a
// module standing on nothing looks like: an edge the mesh has not derived is not an edge.
@@ -0,0 +1,72 @@
package inventory
import (
"encoding/json"
"testing"
"time"
"github.com/novox/mesh-controller/internal/catalogue"
)
// TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack is novox/hq issue 721's review: the newest build of a module
// failed its gate and the mesh put the build before it back. Held names the newest; what runs, and what a merge's build
// is compared with, is the build put back — so a merge whose bytes equal the failed build is never said unchanged.
func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) {
inv := fresh(t)
ctx := t.Context()
manifestOf := func(image string) catalogue.Manifest {
return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{
{"id": "app", "type": "container", "name": "one", "image": image}}}
}
record := func(id, image string, asked time.Time) Build {
t.Helper()
raw, _ := json.Marshal(manifestOf(image))
b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw,
Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}}
if err := inv.RecordBuild(ctx, b); err != nil {
t.Fatal(err)
}
if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit,
Asked: asked}); err != nil {
t.Fatal(err)
}
return b
}
x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111"
y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222"
before := record("x", x, time.Now().Add(-2*time.Hour))
failed := record("y", y, time.Now().Add(-time.Minute))
if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit,
Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil {
t.Fatal(err)
}
if err := inv.RestoreModule(ctx, before); err != nil {
t.Fatal(err)
}
held, err := inv.Held(ctx)
if err != nil {
t.Fatal(err)
}
running, err := inv.RegisteredHeld(ctx)
if err != nil {
t.Fatal(err)
}
if running["one/one"] == y {
t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"])
}
if got := running["one/one"]; got != "" && got != x {
t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x)
}
}
// TestOnlyArtifactsTheRegisteredManifestNamesAreRunning: registeredOnly keeps a held artifact only when its digest is
// in its module's registered manifest.
func TestOnlyArtifactsTheRegisteredManifestNamesAreRunning(t *testing.T) {
held := map[string]string{"one/app": "store/one@sha256:aaaa", "two/app": "store/two@sha256:bbbb", "three/app": "nodigest"}
registered := map[string]string{"one": `{"resources":[{"image":"store/one@sha256:aaaa"}]}`,
"two": `{"resources":[{"image":"store/two@sha256:cccc"}]}`, "three": "nodigest"}
got := registeredOnly(held, registered)
if len(got) != 1 || got["one/app"] == "" {
t.Fatalf("running reads %v, want one/app alone", got)
}
}
+10
View File
@@ -182,6 +182,11 @@ type CheckRequest struct {
// each, from the merge it judged, into the scratch namespace once it has judged and not failed, for the walk to
// take. A build seat from before it builds nothing, and the walk builds as before.
Builds []CheckBuild `json:"builds,omitempty"`
// Running is, keyed "<module>/<artifact>", each artifact of the build the mesh has registered for its module — the
// build its machines are sent (novox/hq issue 721): what a build for the merge is compared with, never the newest
// build, which may be one its gate failed and the mesh put back. A module absent is compared with nothing, and
// moves. A controller from before it sends none, and nothing is said unchanged.
Running map[string]string `json:"running,omitempty"`
// Named are the builds the seat's verdicts of earlier checks of this pull request at this head name, within the
// week check builds are kept (novox/hq issue 703): asked again because only the trunk moved, the check keeps a
// build kept under a module's unchanged key, instead of building it again, only when one of these names its key
@@ -238,6 +243,11 @@ type CheckBuilt struct {
// Kept names the check whose build this check kept instead of building, its key unchanged (novox/hq issue 703):
// a walk takes it as that check's build. Empty when this check built it.
Kept string `json:"kept,omitempty"`
// Same says every artifact came out the same as the build the mesh holds of the module, and SameAs the held
// reference each was compared with (novox/hq issue 721): a walk cut while the mesh still holds those builds leaves
// the module out. Empty when it differs, or could not be compared.
Same string `json:"same,omitempty"`
SameAs map[string]string `json:"same-as,omitempty"`
}
// GroupMember is one other head of a delivery group's composed check.