From dc06af1f5a081a1d73a300070685831b3f478f1f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 21 Oct 2026 19:30:29 +0200 Subject: [PATCH] Leave a module out of a send when the store lacks its image, and say it (issue 1742) --- cmd/mesh-controller/compose_sdk.go | 6 +- .../held_moves_integration_test.go | 92 +++++++ cmd/mesh-controller/image_missing.go | 245 ++++++++++++++++++ cmd/mesh-controller/image_missing_test.go | 125 +++++++++ cmd/mesh-controller/plain_words.go | 7 + cmd/mesh-controller/signals.go | 8 + cmd/mesh-controller/signals_test.go | 7 + cmd/mesh-controller/watchdogs.go | 4 + internal/builder/registry.go | 28 +- internal/builder/registry_test.go | 29 +++ 10 files changed, 546 insertions(+), 5 deletions(-) create mode 100644 cmd/mesh-controller/held_moves_integration_test.go create mode 100644 cmd/mesh-controller/image_missing.go create mode 100644 cmd/mesh-controller/image_missing_test.go diff --git a/cmd/mesh-controller/compose_sdk.go b/cmd/mesh-controller/compose_sdk.go index 8eb0b66f..0a8f6b5e 100644 --- a/cmd/mesh-controller/compose_sdk.go +++ b/cmd/mesh-controller/compose_sdk.go @@ -239,7 +239,11 @@ func markUnresolvable(err error) error { // still names anything foreseen is refused, said, and never sent. func composeSend(ctx context.Context, open *stores, node string) (sendable, error) { declared, _, err := composeSendValidated(ctx, open, node) - return declared, err + if err != nil { + return declared, err + } + // Never a declaration naming an image the store does not hold (novox/hq issue 1742): that module is left out. + return withoutMissingImages(ctx, open, node, declared), nil } // composeSendValidated is composeSend and what the node-engine's validator refuses of the declaration it composed, for the diff --git a/cmd/mesh-controller/held_moves_integration_test.go b/cmd/mesh-controller/held_moves_integration_test.go new file mode 100644 index 00000000..95644ff4 --- /dev/null +++ b/cmd/mesh-controller/held_moves_integration_test.go @@ -0,0 +1,92 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func inventoryUpgrade(why string) inventory.Upgrade { return inventory.Upgrade{Why: why, By: "jochen"} } + +func mustMoveFacts(t *testing.T, rm *recordedMesh) moveFacts { + t.Helper() + f, err := readMoveFacts(t.Context(), rm.open.inventory) + if err != nil { + t.Fatal(err) + } + return f +} + +// 1732/AC8 (integration): a mesh running mailu at c1 on two machines, whose registered c2 recreates a container, is read +// as two machines each holding mailu's move for a person; a bundle-only c3 on top changes nothing of it; after a machine +// is sent c2 (as a push records it) it is no longer among them, and plans that carried c2 say nothing more of it. +func TestMachinesHoldingAMoveForAPersonAreReadFromTheRecords(t *testing.T) { + rm := aRecordedMesh(t, aMailu(true)) + c2 := aMailu(true) + resourceOf(&c2, "imap")["image"] = "registry.invalid/imap@sha256:2" + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + ctx := t.Context() + held, err := gatherHeldMoves(ctx, rm.open.inventory) + if err != nil { + t.Fatal(err) + } + if len(held) != 2 || held[0].node != "anchor" || held[1].node != "laptop" || len(held[0].moves) != 1 || + held[0].moves[0].Module != "mailu" || held[0].moves[0].To != "c2" || !strings.Contains(held[0].moves[0].Why, "recreates") { + t.Fatalf("held %+v", held) + } + if err := rm.open.inventory.RecordSent(ctx, nodeID(t, rm.open, "anchor"), "d-anchor-c2", map[string]string{"mailu": "c2"}); err != nil { + t.Fatal(err) + } + held, err = gatherHeldMoves(ctx, rm.open.inventory) + if err != nil || len(held) != 1 || held[0].node != "laptop" { + t.Fatalf("after anchor was sent c2: %+v (%v)", held, err) + } +} + +// 1732/AC9 (integration): a module a person set to record is no ask: they chose the hold. +func TestAHoldAPersonChoseIsNotAsked(t *testing.T) { + rm := aRecordedMesh(t, aMailu(false)) + c2 := aMailu(false) + c2.Bundles[0].Digest = "sha256:bbb" + rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) + if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventoryUpgrade("the migration week")); err != nil { + t.Fatal(err) + } + held, err := gatherHeldMoves(t.Context(), rm.open.inventory) + if err != nil || len(held) != 0 { + t.Fatalf("a person's own hold is asked: %+v (%v)", held, err) + } + if moves, _ := heldMovesOn(t.Context(), rm.open.inventory, mustMoveFacts(t, rm), "anchor"); len(moves) != 1 || !moves[0].Chosen { + t.Fatalf("status still names it, marked as chosen: %+v", moves) + } +} + +// 1742/AC4 (integration): a registered build naming an image of the store that the store lacks is read as a missing +// image on each machine that runs the module; one the store holds is not; and what the store holds is not asked twice. +func TestARegisteredBuildNamingAnImageTheStoreLacksIsRead(t *testing.T) { + m := aMailu(true) + resourceOf(&m, "imap")["image"] = "artifact-store://mailu/imap@sha256:" + strings.Repeat("e", 64) + rm := aRecordedMesh(t, m) + asked := 0 + lacking := func(_ context.Context, reference string) (bool, error) { asked++; return false, nil } + imagesHeld.Lock() + imagesHeld.held = map[string]bool{} + imagesHeld.Unlock() + got, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking) + if err != nil || len(got) != 2 || got[0].Module != "mailu" || got[0].Node != "anchor" || got[1].Node != "laptop" { + t.Fatalf("missing %+v (%v)", got, err) + } + holding := func(_ context.Context, reference string) (bool, error) { asked++; return true, nil } + if got, err := gatherMissingImages(t.Context(), rm.open.inventory, holding); err != nil || len(got) != 0 { + t.Fatalf("an image the store holds is missing: %+v (%v)", got, err) + } + before := asked + if _, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking); err != nil || asked != before { + t.Fatalf("an image the store was found to hold was asked about again (%d, %v)", asked-before, err) + } +} diff --git a/cmd/mesh-controller/image_missing.go b/cmd/mesh-controller/image_missing.go new file mode 100644 index 00000000..6a894353 --- /dev/null +++ b/cmd/mesh-controller/image_missing.go @@ -0,0 +1,245 @@ +package main + +import ( + "context" + "fmt" + "slices" + "sort" + "strings" + "sync" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-controller/internal/artifacts" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// No declaration is sent that names an image the artifact store does not hold (novox/hq issue 1742). +// +// On 2026-10-21 at 17:13 a send to novox named `mailu/automx@sha256:c62150b8…`; the build record keeps that digest +// ("recent-build") and the store did not hold it. The node-engine removed the running container, then could not pull +// the replacement, and every later apply on novox failed for the next hour while the mail clients' autoconfiguration +// had no backend. The node-engine's half (pull first, replace after) is its own change; this is the controller's: **a +// module whose image the store lacks is left out of the declaration**, as a module whose setting nobody gave is +// (ADR 0163, rule 6) — the node-engine keeps what it has of it and touches none of its containers — and **it is said**, +// row S25: a condition naming the module, the machine and the image, until the store holds the image again (a new build +// of the module, or the copy put back). +// +// Judged only for an image the store's own address names, from the send's own resources; an image of another registry +// is no business of the store's. A store that cannot be asked judges nothing: a send is not refused for a question +// that did not answer. + +// kindImageMissing is S25's kind: module...image-missing. +const kindImageMissing = "image-missing" + +// missingImage is a module held out of a machine's declaration because the store lacks an image it names. +type missingImage struct { + Node, Module, Image string +} + +// storeHolds is whether the store holds the manifest an image reference at its address names. +type storeHolds func(ctx context.Context, reference string) (bool, error) + +// imageInStore is the artifact-store reference of an image the store's address names, "" for any other. +func imageInStore(image, address string) string { + if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") { + return "" + } + reference := catalogue.Recorded(image) + if _, ours := catalogue.InArtifactStore(reference); !ours { + return "" + } + return reference +} + +// holdBackMissingImages leaves out of the declaration every module that names an image the store does not hold, +// and answers what it left out. The module's resources go, it joins LeftOut with the reason, and its build stays the +// one the machine was last sent (`was`). +func holdBackMissingImages(ctx context.Context, node string, s *sendable, address string, holds storeHolds, + was map[string]string) []missingImage { + lacks := map[string]bool{} // reference → the store does not hold it; absent: not asked, or no answer + asked := map[string]bool{} + lacking := map[string]string{} // module → the image + for _, r := range s.Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + image, _ := r["image"].(string) + reference := imageInStore(image, address) + module := s.moduleOf[fmt.Sprint(r["id"])] + if reference == "" || module == "" || lacking[module] != "" { + continue + } + if !asked[reference] { + asked[reference] = true + held, err := holds(ctx, reference) + // empty-on-error: the store did not answer, so nothing is held back for a question unanswered + lacks[reference] = err == nil && !held + } + if lacks[reference] { + lacking[module] = image + } + } + if len(lacking) == 0 { + return nil + } + kept := s.Resources[:0:0] + for _, r := range s.Resources { + if _, out := lacking[s.moduleOf[fmt.Sprint(r["id"])]]; !out { + kept = append(kept, r) + } + } + s.Resources = kept + if s.leftOutWhy == nil { + s.leftOutWhy = map[string]string{} + } + var found []missingImage + for module, image := range lacking { + s.leftOutWhy[module] = "its image " + image + " is not in the artifact store" + if !slices.Contains(s.LeftOut, module) { + s.LeftOut = append(s.LeftOut, module) + } + if s.Builds != nil { + if prior, sent := was[module]; sent { + s.Builds[module] = prior + } else { + delete(s.Builds, module) + } + } + found = append(found, missingImage{Node: node, Module: module, Image: image}) + } + sort.Strings(s.LeftOut) + sort.Slice(found, func(i, j int) bool { return found[i].Module < found[j].Module }) + return found +} + +// withoutMissingImages is composeSend's last step: the store asked about the images the declaration names, the modules +// whose image it lacks left out and kept in the ledger S25 reads. A store that cannot be found or asked leaves the +// declaration as composed. +func withoutMissingImages(ctx context.Context, open *stores, node string, s sendable) sendable { + inv := open.inventory + shelf, err := inv.Catalogue(ctx) + if err != nil { + return s + } + address, err := artifactStoreAddress(ctx, inv, shelf, node) + if err != nil || address == "" { + return s + } + sent, known, err := inv.SentBuilds(ctx, node) + if err != nil || !known { + sent = nil + } + store := artifacts.Store{Address: address} + found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, sent) + for _, m := range found { + fmt.Printf("%s: %s is left out of the declaration: its image %s is not in the artifact store\n", node, m.Module, m.Image) + } + return s +} + +// imageFacts are the images S25 raises: a module whose registered build names an image the store lacks, on each machine +// that runs it. +type imageFacts struct { + missingImage +} + +// imagesHeld remembers the images the store was found to hold: a stored image is named by its digest and stays what it +// is, so it is asked about once; one it lacks is asked again at every look. +var imagesHeld = struct { + sync.Mutex + held map[string]bool +}{held: map[string]bool{}} + +// gatherMissingImages is every machine's module whose registered build — the one a send would carry — names an image of +// the artifact store that the store does not hold. Read from the catalogue and the store, never from what a send did: +// a push is a command of its own, and the serving controller's watchdog hears nothing of it. +func gatherMissingImages(ctx context.Context, inv *inventory.Inventory, holds storeHolds) ([]imageFacts, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, fmt.Errorf("the catalogue cannot be read: %w", err) + } + var out []imageFacts + for _, module := range sortedKeysOf(modulesOf(shelf)) { + for _, r := range shelf[module].Resources { + if fmt.Sprint(r["type"]) != "container" { + continue + } + image, _ := r["image"].(string) + if _, ours := catalogue.InArtifactStore(image); !ours || !strings.Contains(image, "@sha256:") { + continue + } + imagesHeld.Lock() + known := imagesHeld.held[image] + imagesHeld.Unlock() + if !known { + held, err := holds(ctx, image) + if err != nil { + return nil, fmt.Errorf("the artifact store cannot be asked about %s: %w", image, err) + } + if held { + imagesHeld.Lock() + imagesHeld.held[image] = true + imagesHeld.Unlock() + continue + } + } else { + continue + } + running, err := inv.Running(ctx, module) + if err != nil { + return nil, err + } + for _, node := range running { + out = append(out, imageFacts{missingImage{Node: node, Module: module, Image: image}}) + } + } + } + return out, nil +} + +// modulesOf is the shelf's modules as a map of names, for sortedKeysOf. +func modulesOf(shelf map[string]catalogue.Manifest) map[string]string { + out := make(map[string]string, len(shelf)) + for m := range shelf { + out[m] = m + } + return out +} + +// watchMissingImages is S25: a module left out of a machine's declaration because the store lacks its image, said at +// once. The condition ends when the store holds the image, or a send names another. +func watchMissingImages(f *signalFacts) []conditions.Observation { + var out []conditions.Observation + for _, m := range f.images { + out = append(out, conditions.Observation{Scope: conditions.ScopeModule, ID: m.Module + "." + m.Node, Kind: kindImageMissing, + Machine: m.Node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names %s, which the artifact store does not hold; "+ + "the running container is untouched", m.Module, m.Node, m.Image), + Said: "image " + m.Image + " is not in the store", + Headline: clipWords(conditions.Capital(m.Module)+" is not updated on "+m.Node, conditions.HeadlineMax), + Explanation: clipWords(fmt.Sprintf("The new version of %s names a container image that the mesh's store does not "+ + "hold, so it was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.", + m.Module, m.Node, m.Node), conditions.ExplanationMax), + Needs: fmt.Sprintf("build %s again; the details name the image.", m.Module), + Resolved: "Resolved: the store holds the image of " + m.Module + " again"}) + } + return out +} + +// gatherMissingImagesHere is gatherMissingImages asking the mesh's own artifact store; nothing to ask where the mesh has +// none on its network yet. +func gatherMissingImagesHere(ctx context.Context, inv *inventory.Inventory) ([]imageFacts, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, fmt.Errorf("the catalogue cannot be read: %w", err) + } + address, err := artifactStoreAddress(ctx, inv, shelf, "") + if err != nil { + return nil, err + } + if address == "" { + return nil, nil + } + return gatherMissingImages(ctx, inv, artifacts.Store{Address: address}.HoldsManifest) +} diff --git a/cmd/mesh-controller/image_missing_test.go b/cmd/mesh-controller/image_missing_test.go new file mode 100644 index 00000000..86362ea9 --- /dev/null +++ b/cmd/mesh-controller/image_missing_test.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" +) + +// **No declaration is sent that names an image the artifact store does not hold** (novox/hq issue 1742). + +const storeAt = "novox.internal:5100" + +// automx is the image the send of 2026-10-21 17:13 named and the store lacked; the others are held. +const ( + automxImage = storeAt + "/mailu/automx@sha256:c62150b8e2cbcc18cb911b098c2877f928761a1e6251667f39ea18251ea35591" + imapImage = storeAt + "/mailu/imap@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +) + +func aMailSend() sendable { + return sendable{ + Resources: []map[string]any{ + {"id": "mailu.automx", "type": "container", "image": automxImage}, + {"id": "mailu.automx-config", "type": "file", "path": "/etc/automx.conf"}, + {"id": "mailu.imap", "type": "container", "image": imapImage}, + {"id": "other.web", "type": "container", "image": "docker.io/library/nginx@sha256:" + strings.Repeat("b", 64)}, + {"id": "dns.dnsmasq", "type": "container", "image": storeAt + "/dnsmasq/main@sha256:" + strings.Repeat("c", 64)}, + }, + moduleOf: map[string]string{"mailu.automx": "automx", "mailu.automx-config": "automx", "mailu.imap": "mailu", + "other.web": "other", "dns.dnsmasq": "dnsmasq"}, + Builds: map[string]string{"automx": "b2f5ebb2new", "mailu": "m2", "other": "o1", "dnsmasq": "d1"}, + } +} + +func storeHolding(missing ...string) (storeHolds, *[]string) { + var asked []string + return func(_ context.Context, reference string) (bool, error) { + asked = append(asked, reference) + for _, m := range missing { + if strings.Contains(reference, m) { + return false, nil + } + } + return true, nil + }, &asked +} + +// 1742/AC1: given a send naming automx's image, which the store does not hold, when the declaration is judged, then +// every resource of automx is out of it, automx is among the modules left out with the reason, its build is the one the +// machine was last sent, and every other module — an image of another registry among them — is as composed. +func TestAModuleWhoseImageTheStoreLacksIsLeftOutOfTheSend(t *testing.T) { + s := aMailSend() + holds, asked := storeHolding("mailu/automx@") + found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, holds, map[string]string{"automx": "b2f5ebb2old"}) + if len(found) != 1 || found[0].Module != "automx" || found[0].Node != "novox" || found[0].Image != automxImage { + t.Fatalf("found %+v", found) + } + var ids []string + for _, r := range s.Resources { + ids = append(ids, r["id"].(string)) + } + if got := strings.Join(ids, ","); got != "mailu.imap,other.web,dns.dnsmasq" { + t.Errorf("the declaration keeps %s", got) + } + if len(s.LeftOut) != 1 || s.LeftOut[0] != "automx" || !strings.Contains(s.leftOutWhy["automx"], "not in the artifact store") { + t.Errorf("left out %v because %v", s.LeftOut, s.leftOutWhy) + } + if s.Builds["automx"] != "b2f5ebb2old" || s.Builds["mailu"] != "m2" || s.Builds["other"] != "o1" { + t.Errorf("builds %v", s.Builds) + } + for _, a := range *asked { + if strings.Contains(a, "docker.io") { + t.Errorf("the store was asked about an image of another registry: %s", a) + } + } +} + +// 1742/AC2: given a module never sent to the machine, when its image is missing, then it has no build in the record of the +// send; given the store does not answer, then nothing is held back for a question unanswered. +func TestAnUnansweredStoreHoldsNothingBackAndANewModuleHasNoBuild(t *testing.T) { + s := aMailSend() + holds, _ := storeHolding("mailu/automx@") + found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, holds, nil) + if len(found) != 1 { + t.Fatalf("%+v", found) + } + if _, has := s.Builds["automx"]; has { + t.Errorf("a module never sent is recorded as sent: %v", s.Builds) + } + s = aMailSend() + silent := func(context.Context, string) (bool, error) { return false, errors.New("registry: connection refused") } + if found := holdBackMissingImages(t.Context(), "novox", &s, storeAt, silent, nil); len(found) != 0 || len(s.LeftOut) != 0 || + len(s.Resources) != 5 { + t.Errorf("an unanswered store held back %+v, left out %v", found, s.LeftOut) + } +} + +// 1742/AC3: a module left out for its image raises a condition for the operator naming the module, the machine and the +// image in the evidence, plain words, and no answer to give; a mesh whose images are all held raises nothing. +func TestAMissingImageIsRaisedForTheOperator(t *testing.T) { + f := calm(time.Date(2026, 10, 21, 17, 40, 0, 0, time.UTC)) + if got := watchMissingImages(f); len(got) != 0 { + t.Fatalf("%+v", got) + } + f.images = []imageFacts{{missingImage{Node: "novox", Module: "automx", Image: automxImage}}} + got := watchMissingImages(f) + if len(got) != 1 { + t.Fatalf("%+v", got) + } + o := got[0] + if o.Kind != "image-missing" || o.Key() != "module.automx.novox.image-missing" || o.Resolver != conditions.ResolverOperator || + o.Severity != conditions.Warning || o.Machine != "novox" { + t.Fatalf("%+v (%s)", o, o.Key()) + } + if !strings.Contains(o.Said, "c62150b8") || !strings.Contains(o.Headline+o.Explanation, "automx") { + t.Errorf("words: %q / %q / %q", o.Headline, o.Explanation, o.Said) + } + w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} + if why, ok := conditions.PlainWords(w, "novox"); !ok { + t.Errorf("not plain: %s", why) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 03e717a4..fce97830 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -462,6 +462,13 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "A new version of a module would restart a container or touch its data, so it waits for you.", Resolved: "Resolved: the update is no longer waiting"} }), + kindImageMissing: worded(func(o conditions.Observation) words { + return words{Headline: "A module is not updated", + Explanation: "The new version of a module names a container image that the mesh's store does not hold, so it " + + "was not sent. What runs is untouched.", + Needs: "build the module again; the details name the image.", + Resolved: "Resolved: the store holds the image again"} + }), "ask-lost": worded(func(o conditions.Observation) words { return words{Headline: "A build has no result", Explanation: "A build was asked for and never answered. It is asked again with the next delivery.", diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index f1e02351..76e65509 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -252,6 +252,14 @@ var signalsTable = []signalRow{ Kind: kindMovesHeld, Severity: conditions.Warning, Phase: 3, needs: func(f *signalFacts) error { return f.heldErr }, watch: watchMovesHeld, newest: func(f *signalFacts) time.Time { return time.Time{} }}, + {Row: "S25", Signal: "a module left out of a send because the artifact store lacks an image it names", + Emitter: "controller's send", Trigger: "each send whose declaration names an image of the store that the store does not hold (novox/hq issue 1742)", + Bound: "none: raised on the first look at a registered build that names such an image, naming the module, each " + + "machine that runs it and the image; every send leaves that module out; the store is asked again at each " + + "look and the condition ends when it holds the image", + Kind: kindImageMissing, Severity: conditions.Warning, Phase: 3, + needs: func(f *signalFacts) error { return f.imagesErr }, watch: watchMissingImages, + newest: func(f *signalFacts) time.Time { return time.Time{} }}, {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index d583d35a..9bf41d09 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -177,6 +177,13 @@ var suppressions = map[string]suppression{ inside: func(f *signalFacts) { f.held = nil }, past: func(f *signalFacts) { f.held = []heldFacts{aHeldMailu()} }, }, + // A registered build naming an image the artifact store lacks is raised at the first look (novox/hq issue 1742). + "S25": { + inside: func(f *signalFacts) { f.images = nil }, + past: func(f *signalFacts) { + f.images = []imageFacts{{missingImage{Node: "anchor", Module: "automx", Image: automxImage}}} + }, + }, // A batch queued behind an open walk past that walk's bound, named. "S19": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 462a80e1..c714678d 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -61,6 +61,9 @@ type signalFacts struct { // held are the machines with moves of recorded modules waiting for a person's push (S24, novox/hq issue 1732). held []heldFacts heldErr error + // images are the modules left out of a send because the artifact store lacks an image they name (S25, issue 1742). + images []imageFacts + imagesErr error // refusedSends are the sends a machine refused within the hour for the generation they were composed // from, each naming its sender (S20, novox/hq issue 234). refusedSends []inventory.Send @@ -361,6 +364,7 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now) f.held, f.heldErr = gatherHeldMoves(ctx, inv) + f.images, f.imagesErr = gatherMissingImagesHere(ctx, inv) f.loop, f.loopErr = w.gatherLoop() f.asks, f.asksErr = w.gatherAsks(ctx, inv) if open, err := w.keeper.Open(ctx); err != nil { diff --git a/internal/builder/registry.go b/internal/builder/registry.go index 4f7ecd11..95918550 100644 --- a/internal/builder/registry.go +++ b/internal/builder/registry.go @@ -59,15 +59,35 @@ func (r Registry) PublishImage(ctx context.Context, localTag, repository string) }); err != nil { return "", err } - out, err := r.Run(ctx, "", "docker", "inspect", "--format", "{{index .RepoDigests 0}}", remote) + // Every digest the daemon lists for the image, one per line: the image may have been pushed under other names before + // (a check's scratch copy, another registry), and the first of them is not the one this push made (novox/hq issue + // 1742: a build recorded a digest the store never held). + out, err := r.Run(ctx, "", "docker", "inspect", "--format", "{{range .RepoDigests}}{{println .}}{{end}}", remote) if err != nil { return "", fmt.Errorf("pushed %s and cannot read back what it was pinned as: %w", remote, err) } - pinned := strings.TrimSpace(out) - if !strings.Contains(pinned, "@sha256:") { + first, pinned := "", "" + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if first == "" { + first = line + } + if strings.HasPrefix(line, remote+"@sha256:") { + pinned = line + break + } + } + if pinned == "" { + if strings.Contains(first, "@sha256:") { + return "", fmt.Errorf("pushed %s, but the daemon lists no digest for that name (only %q): the digest of another "+ + "repository is not what this registry serves", remote, first) + } // A tag is not a pin. It can be made to point at something else, and this file is applied // on machines with no mesh to ask about anything (novox/hq ADR 0006). - return "", fmt.Errorf("%s came back as %q, which is not pinned by digest", remote, pinned) + return "", fmt.Errorf("%s came back as %q, which is not pinned by digest", remote, first) } return pinned, nil } diff --git a/internal/builder/registry_test.go b/internal/builder/registry_test.go index 1e3a440d..69ff7c2b 100644 --- a/internal/builder/registry_test.go +++ b/internal/builder/registry_test.go @@ -252,3 +252,32 @@ func TestAPushedImageComesBackPinned(t *testing.T) { t.Fatalf("it did not tag and then push: %v", ran) } } + +// 1742: an image pushed under other names before lists their digests too, and the digest recorded is the one of the name +// this push made — never the first listed, which may be another repository's. +func TestThePinnedDigestIsTheOneOfTheNamePushed(t *testing.T) { + want := "registry.invalid/shell/server@sha256:" + strings.Repeat("b", 64) + earlier := "registry.invalid/scratch/shell/server@sha256:" + strings.Repeat("a", 64) + r := Registry{Address: "registry.invalid", Run: func( + _ context.Context, _, name string, args ...string) (string, error) { + if name == "docker" && len(args) > 0 && args[0] == "inspect" { + return earlier + "\n" + want + "\n", nil + } + return "", nil + }} + got, err := r.PublishImage(context.Background(), "local", "shell/server") + if err != nil || got != want { + t.Fatalf("pinned %q (%v); want %q", got, err, want) + } + // Only another repository's digest is listed: the push is not taken for pinned. + r.Run = func(_ context.Context, _, name string, args ...string) (string, error) { + if name == "docker" && len(args) > 0 && args[0] == "inspect" { + return earlier + "\n", nil + } + return "", nil + } + if _, err := r.PublishImage(context.Background(), "local", "shell/server"); err == nil || + !strings.Contains(err.Error(), "no digest for that name") { + t.Fatalf("a digest of another repository was taken for the push's own: %v", err) + } +}