246 lines
9.4 KiB
Go
246 lines
9.4 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"slices"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
"github.com/novox/mesh-controller/internal/artifacts"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// No declaration is sent that names an image the artifact store does not hold (novox/hq issue 1742).
|
|
//
|
|
// On 2026-10-21 at 17:13 a send to novox named `mailu/automx@sha256:c62150b8…`; the build record keeps that digest
|
|
// ("recent-build") and the store did not hold it. The node-engine removed the running container, then could not pull
|
|
// the replacement, and every later apply on novox failed for the next hour while the mail clients' autoconfiguration
|
|
// had no backend. The node-engine's half (pull first, replace after) is its own change; this is the controller's: **a
|
|
// module whose image the store lacks is left out of the declaration**, as a module whose setting nobody gave is
|
|
// (ADR 0163, rule 6) — the node-engine keeps what it has of it and touches none of its containers — and **it is said**,
|
|
// row S25: a condition naming the module, the machine and the image, until the store holds the image again (a new build
|
|
// of the module, or the copy put back).
|
|
//
|
|
// Judged only for an image the store's own address names, from the send's own resources; an image of another registry
|
|
// is no business of the store's. A store that cannot be asked judges nothing: a send is not refused for a question
|
|
// that did not answer.
|
|
|
|
// kindImageMissing is S25's kind: module.<module>.<machine>.image-missing.
|
|
const kindImageMissing = "image-missing"
|
|
|
|
// missingImage is a module held out of a machine's declaration because the store lacks an image it names.
|
|
type missingImage struct {
|
|
Node, Module, Image string
|
|
}
|
|
|
|
// storeHolds is whether the store holds the manifest an image reference at its address names.
|
|
type storeHolds func(ctx context.Context, reference string) (bool, error)
|
|
|
|
// imageInStore is the artifact-store reference of an image the store's address names, "" for any other.
|
|
func imageInStore(image, address string) string {
|
|
if address == "" || !strings.HasPrefix(image, address+"/") || !strings.Contains(image, "@sha256:") {
|
|
return ""
|
|
}
|
|
reference := catalogue.Recorded(image)
|
|
if _, ours := catalogue.InArtifactStore(reference); !ours {
|
|
return ""
|
|
}
|
|
return reference
|
|
}
|
|
|
|
// holdBackMissingImages leaves out of the declaration every module that names an image the store does not hold,
|
|
// and answers what it left out. The module's resources go, it joins LeftOut with the reason, and its build stays the
|
|
// one the machine was last sent (`was`).
|
|
func holdBackMissingImages(ctx context.Context, node string, s *sendable, address string, holds storeHolds,
|
|
was map[string]string) []missingImage {
|
|
lacks := map[string]bool{} // reference → the store does not hold it; absent: not asked, or no answer
|
|
asked := map[string]bool{}
|
|
lacking := map[string]string{} // module → the image
|
|
for _, r := range s.Resources {
|
|
if fmt.Sprint(r["type"]) != "container" {
|
|
continue
|
|
}
|
|
image, _ := r["image"].(string)
|
|
reference := imageInStore(image, address)
|
|
module := s.moduleOf[fmt.Sprint(r["id"])]
|
|
if reference == "" || module == "" || lacking[module] != "" {
|
|
continue
|
|
}
|
|
if !asked[reference] {
|
|
asked[reference] = true
|
|
held, err := holds(ctx, reference)
|
|
// empty-on-error: the store did not answer, so nothing is held back for a question unanswered
|
|
lacks[reference] = err == nil && !held
|
|
}
|
|
if lacks[reference] {
|
|
lacking[module] = image
|
|
}
|
|
}
|
|
if len(lacking) == 0 {
|
|
return nil
|
|
}
|
|
kept := s.Resources[:0:0]
|
|
for _, r := range s.Resources {
|
|
if _, out := lacking[s.moduleOf[fmt.Sprint(r["id"])]]; !out {
|
|
kept = append(kept, r)
|
|
}
|
|
}
|
|
s.Resources = kept
|
|
if s.leftOutWhy == nil {
|
|
s.leftOutWhy = map[string]string{}
|
|
}
|
|
var found []missingImage
|
|
for module, image := range lacking {
|
|
s.leftOutWhy[module] = "its image " + image + " is not in the artifact store"
|
|
if !slices.Contains(s.LeftOut, module) {
|
|
s.LeftOut = append(s.LeftOut, module)
|
|
}
|
|
if s.Builds != nil {
|
|
if prior, sent := was[module]; sent {
|
|
s.Builds[module] = prior
|
|
} else {
|
|
delete(s.Builds, module)
|
|
}
|
|
}
|
|
found = append(found, missingImage{Node: node, Module: module, Image: image})
|
|
}
|
|
sort.Strings(s.LeftOut)
|
|
sort.Slice(found, func(i, j int) bool { return found[i].Module < found[j].Module })
|
|
return found
|
|
}
|
|
|
|
// withoutMissingImages is composeSend's last step: the store asked about the images the declaration names, the modules
|
|
// whose image it lacks left out and kept in the ledger S25 reads. A store that cannot be found or asked leaves the
|
|
// declaration as composed.
|
|
func withoutMissingImages(ctx context.Context, open *stores, node string, s sendable) sendable {
|
|
inv := open.inventory
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return s
|
|
}
|
|
address, err := artifactStoreAddress(ctx, inv, shelf, node)
|
|
if err != nil || address == "" {
|
|
return s
|
|
}
|
|
sent, known, err := inv.SentBuilds(ctx, node)
|
|
if err != nil || !known {
|
|
sent = nil
|
|
}
|
|
store := artifacts.Store{Address: address}
|
|
found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, sent)
|
|
for _, m := range found {
|
|
fmt.Printf("%s: %s is left out of the declaration: its image %s is not in the artifact store\n", node, m.Module, m.Image)
|
|
}
|
|
return s
|
|
}
|
|
|
|
// imageFacts are the images S25 raises: a module whose registered build names an image the store lacks, on each machine
|
|
// that runs it.
|
|
type imageFacts struct {
|
|
missingImage
|
|
}
|
|
|
|
// imagesHeld remembers the images the store was found to hold: a stored image is named by its digest and stays what it
|
|
// is, so it is asked about once; one it lacks is asked again at every look.
|
|
var imagesHeld = struct {
|
|
sync.Mutex
|
|
held map[string]bool
|
|
}{held: map[string]bool{}}
|
|
|
|
// gatherMissingImages is every machine's module whose registered build — the one a send would carry — names an image of
|
|
// the artifact store that the store does not hold. Read from the catalogue and the store, never from what a send did:
|
|
// a push is a command of its own, and the serving controller's watchdog hears nothing of it.
|
|
func gatherMissingImages(ctx context.Context, inv *inventory.Inventory, holds storeHolds) ([]imageFacts, error) {
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
|
}
|
|
var out []imageFacts
|
|
for _, module := range sortedKeysOf(modulesOf(shelf)) {
|
|
for _, r := range shelf[module].Resources {
|
|
if fmt.Sprint(r["type"]) != "container" {
|
|
continue
|
|
}
|
|
image, _ := r["image"].(string)
|
|
if _, ours := catalogue.InArtifactStore(image); !ours || !strings.Contains(image, "@sha256:") {
|
|
continue
|
|
}
|
|
imagesHeld.Lock()
|
|
known := imagesHeld.held[image]
|
|
imagesHeld.Unlock()
|
|
if !known {
|
|
held, err := holds(ctx, image)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the artifact store cannot be asked about %s: %w", image, err)
|
|
}
|
|
if held {
|
|
imagesHeld.Lock()
|
|
imagesHeld.held[image] = true
|
|
imagesHeld.Unlock()
|
|
continue
|
|
}
|
|
} else {
|
|
continue
|
|
}
|
|
running, err := inv.Running(ctx, module)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range running {
|
|
out = append(out, imageFacts{missingImage{Node: node, Module: module, Image: image}})
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// modulesOf is the shelf's modules as a map of names, for sortedKeysOf.
|
|
func modulesOf(shelf map[string]catalogue.Manifest) map[string]string {
|
|
out := make(map[string]string, len(shelf))
|
|
for m := range shelf {
|
|
out[m] = m
|
|
}
|
|
return out
|
|
}
|
|
|
|
// watchMissingImages is S25: a module left out of a machine's declaration because the store lacks its image, said at
|
|
// once. The condition ends when the store holds the image, or a send names another.
|
|
func watchMissingImages(f *signalFacts) []conditions.Observation {
|
|
var out []conditions.Observation
|
|
for _, m := range f.images {
|
|
out = append(out, conditions.Observation{Scope: conditions.ScopeModule, ID: m.Module + "." + m.Node, Kind: kindImageMissing,
|
|
Machine: m.Node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator,
|
|
Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+
|
|
"not hold (the evidence names it); the running container is untouched", m.Module, m.Node),
|
|
Said: "image " + m.Image + " is not in the store",
|
|
Headline: clipWords(conditions.Capital(m.Module)+" is not updated on "+m.Node, conditions.HeadlineMax),
|
|
Explanation: clipWords(fmt.Sprintf("The new version of %s names a container image that the mesh's store does not "+
|
|
"hold, so it was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.",
|
|
m.Module, m.Node, m.Node), conditions.ExplanationMax),
|
|
Needs: fmt.Sprintf("build %s again; the details name the image.", m.Module),
|
|
Resolved: "Resolved: the store holds the image of " + m.Module + " again"})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// gatherMissingImagesHere is gatherMissingImages asking the mesh's own artifact store; nothing to ask where the mesh has
|
|
// none on its network yet.
|
|
func gatherMissingImagesHere(ctx context.Context, inv *inventory.Inventory) ([]imageFacts, error) {
|
|
shelf, err := inv.Catalogue(ctx)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("the catalogue cannot be read: %w", err)
|
|
}
|
|
address, err := artifactStoreAddress(ctx, inv, shelf, "")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if address == "" {
|
|
return nil, nil
|
|
}
|
|
return gatherMissingImages(ctx, inv, artifacts.Store{Address: address}.HoldsManifest)
|
|
}
|