package main import ( "context" "fmt" "slices" "sort" "strings" "sync" "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/artifacts" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" ) // No declaration is sent that names an image the artifact store does not hold (novox/hq issue 1742). // // On 2026-10-21 at 17:13 a send to novox named `mailu/automx@sha256:c62150b8…`; the build record keeps that digest // ("recent-build") and the store did not hold it. The node-engine removed the running container, then could not pull // the replacement, and every later apply on novox failed for the next hour while the mail clients' autoconfiguration // had no backend. The node-engine's half (pull first, replace after) is its own change; this is the controller's: **a // module whose image the store lacks is left out of the declaration**, as a module whose setting nobody gave is // (ADR 0163, rule 6) — the node-engine keeps what it has of it and touches none of its containers — and **it is said**, // row S25: a condition naming the module, the machine and the image, until the store holds the image again (a new build // of the module, or the copy put back). // // Judged only for an image the store's own address names, from the send's own resources; an image of another registry // is no business of the store's. A store that cannot be asked judges nothing: a send is not refused for a question // that did not answer. // kindImageMissing is S25's kind: module...image-missing. const kindImageMissing = "image-missing" // missingImage is a module held out of a machine's declaration because the store lacks an image it names. type missingImage struct { Node, Module, Image string } // storeHolds is whether the store holds the manifest an image reference at its address names. type storeHolds func(ctx context.Context, reference string) (bool, error) // imageInStore is the artifact-store reference of an image the store's address names, "" for any other. func imageInStore(image, address string) string { if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") { return "" } reference := catalogue.Recorded(image) if _, ours := catalogue.InArtifactStore(reference); !ours { return "" } return reference } // holdBackMissingImages leaves out of the declaration every module that names an image the store does not hold, // and answers what it left out. The module's resources go, it joins LeftOut with the reason, and its build stays the // one the machine was last sent (`was`). func holdBackMissingImages(ctx context.Context, node string, s *sendable, address string, holds storeHolds, was map[string]string) []missingImage { lacks := map[string]bool{} // reference → the store does not hold it; absent: not asked, or no answer asked := map[string]bool{} lacking := map[string]string{} // module → the image for _, r := range s.Resources { if fmt.Sprint(r["type"]) != "container" { continue } image, _ := r["image"].(string) reference := imageInStore(image, address) module := s.moduleOf[fmt.Sprint(r["id"])] if reference == "" || module == "" || lacking[module] != "" { continue } if !asked[reference] { asked[reference] = true held, err := holds(ctx, reference) // empty-on-error: the store did not answer, so nothing is held back for a question unanswered lacks[reference] = err == nil && !held } if lacks[reference] { lacking[module] = image } } if len(lacking) == 0 { return nil } kept := s.Resources[:0:0] for _, r := range s.Resources { if _, out := lacking[s.moduleOf[fmt.Sprint(r["id"])]]; !out { kept = append(kept, r) } } s.Resources = kept if s.leftOutWhy == nil { s.leftOutWhy = map[string]string{} } var found []missingImage for module, image := range lacking { s.leftOutWhy[module] = "its image " + image + " is not in the artifact store" if !slices.Contains(s.LeftOut, module) { s.LeftOut = append(s.LeftOut, module) } if s.Builds != nil { if prior, sent := was[module]; sent { s.Builds[module] = prior } else { delete(s.Builds, module) } } found = append(found, missingImage{Node: node, Module: module, Image: image}) } sort.Strings(s.LeftOut) sort.Slice(found, func(i, j int) bool { return found[i].Module < found[j].Module }) return found } // withoutMissingImages is composeSend's last step: the store asked about the images the declaration names, the modules // whose image it lacks left out and kept in the ledger S25 reads. A store that cannot be found or asked leaves the // declaration as composed. func withoutMissingImages(ctx context.Context, open *stores, node string, s sendable) sendable { inv := open.inventory shelf, err := inv.Catalogue(ctx) if err != nil { return s } address, err := artifactStoreAddress(ctx, inv, shelf, node) if err != nil || address == "" { return s } sent, known, err := inv.SentBuilds(ctx, node) if err != nil || !known { sent = nil } store := artifacts.Store{Address: address} found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, sent) for _, m := range found { fmt.Printf("%s: %s is left out of the declaration: its image %s is not in the artifact store\n", node, m.Module, m.Image) } return s } // imageFacts are the images S25 raises: a module whose registered build names an image the store lacks, on each machine // that runs it. type imageFacts struct { missingImage } // imagesHeld remembers the images the store was found to hold: a stored image is named by its digest and stays what it // is, so it is asked about once; one it lacks is asked again at every look. var imagesHeld = struct { sync.Mutex held map[string]bool }{held: map[string]bool{}} // gatherMissingImages is every machine's module whose registered build — the one a send would carry — names an image of // the artifact store that the store does not hold. Read from the catalogue and the store, never from what a send did: // a push is a command of its own, and the serving controller's watchdog hears nothing of it. func gatherMissingImages(ctx context.Context, inv *inventory.Inventory, holds storeHolds) ([]imageFacts, error) { shelf, err := inv.Catalogue(ctx) if err != nil { return nil, fmt.Errorf("the catalogue cannot be read: %w", err) } var out []imageFacts for _, module := range sortedKeysOf(modulesOf(shelf)) { for _, r := range shelf[module].Resources { if fmt.Sprint(r["type"]) != "container" { continue } image, _ := r["image"].(string) if _, ours := catalogue.InArtifactStore(image); !ours || !strings.Contains(image, "@sha256:") { continue } imagesHeld.Lock() known := imagesHeld.held[image] imagesHeld.Unlock() if !known { held, err := holds(ctx, image) if err != nil { return nil, fmt.Errorf("the artifact store cannot be asked about %s: %w", image, err) } if held { imagesHeld.Lock() imagesHeld.held[image] = true imagesHeld.Unlock() continue } } else { continue } running, err := inv.Running(ctx, module) if err != nil { return nil, err } for _, node := range running { out = append(out, imageFacts{missingImage{Node: node, Module: module, Image: image}}) } } } return out, nil } // modulesOf is the shelf's modules as a map of names, for sortedKeysOf. func modulesOf(shelf map[string]catalogue.Manifest) map[string]string { out := make(map[string]string, len(shelf)) for m := range shelf { out[m] = m } return out } // watchMissingImages is S25: a module left out of a machine's declaration because the store lacks its image, said at // once. The condition ends when the store holds the image, or a send names another. func watchMissingImages(f *signalFacts) []conditions.Observation { var out []conditions.Observation for _, m := range f.images { out = append(out, conditions.Observation{Scope: conditions.ScopeModule, ID: m.Module + "." + m.Node, Kind: kindImageMissing, Machine: m.Node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+ "not hold (the evidence names it); the running container is untouched", m.Module, m.Node), Said: "image " + m.Image + " is not in the store", Headline: clipWords(conditions.Capital(m.Module)+" is not updated on "+m.Node, conditions.HeadlineMax), Explanation: clipWords(fmt.Sprintf("The new version of %s names a container image that the mesh's store does not "+ "hold, so it was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.", m.Module, m.Node, m.Node), conditions.ExplanationMax), Needs: fmt.Sprintf("build %s again; the details name the image.", m.Module), Resolved: "Resolved: the store holds the image of " + m.Module + " again"}) } return out } // gatherMissingImagesHere is gatherMissingImages asking the mesh's own artifact store; nothing to ask where the mesh has // none on its network yet. func gatherMissingImagesHere(ctx context.Context, inv *inventory.Inventory) ([]imageFacts, error) { shelf, err := inv.Catalogue(ctx) if err != nil { return nil, fmt.Errorf("the catalogue cannot be read: %w", err) } address, err := artifactStoreAddress(ctx, inv, shelf, "") if err != nil { return nil, err } if address == "" { return nil, nil } return gatherMissingImages(ctx, inv, artifacts.Store{Address: address}.HoldsManifest) }