internal/composer/composer.go

@@ -15,6 +15,7 @@ import (
 	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
 	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
 	"git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay"
+	"maps"
 	"slices"
 	"sort"
 	"strings"
@@ -162,6 +163,8 @@ func runCase(r Request, c Case, depth string) (CaseVerdict, error) {
 	if err != nil {
 		return CaseVerdict{}, err
 	}
+	held := base
+	base, awaited := overrule(base, c, cv.Manifests, shelf)
 	assignments := map[string]*meshmoduleassigner.Assignments{}
 	for node, raw := range c.Assignments {
 		if string(raw) == "null" {
@@ -189,9 +192,17 @@ func runCase(r Request, c Case, depth string) (CaseVerdict, error) {
 		shelf[name] = m
 	}
 	// A module the input carries as its node-engine took it in from its build (SentInput) is composed so: the bus
-	// reader composes it from its build, never from the indexer's manifest a case gives.
+	// reader composes it from its build, never from the indexer's manifest a case gives. One the case overrules the
+	// node's hold on is composed at the build the node would compose it at (Input.Holds), else from the case's manifest.
 	for name, m := range base.Catalogue {
-		shelf[name] = m
+		if _, over := awaited[name]; !over {
+			shelf[name] = m
+		}
+	}
+	for name, hold := range base.Holds {
+		if _, over := awaited[name]; over && hold.Built != nil {
+			shelf[name] = *hold.Built
+		}
 	}
 	// A module the input's node-engine holds as refused (compose.Input.Refused: its verdict, or bytes that do not decode) is
 	// held here as there, and its read says why, whatever the case's own verdicts: the check then names the cause the node
@@ -225,11 +236,134 @@ func runCase(r Request, c Case, depth string) (CaseVerdict, error) {
 	}
 	cv.Machines = map[string]Machine{}
 	for _, node := range r.Machines {
-		cv.Machines[node] = machine(in, node, depth, r.Declarations)
+		m := machine(in, node, depth, r.Declarations)
+		if words := awaitedOn(in, node, awaited); words != "" {
+			// Composed without the build it runs at: named per node and module, never a silent pass (issue 1921). The
+			// machine is then not judged, as for a node whose running state its engine does not send (ADR 0593 §9). Where
+			// it does not compose without that build (a tool runner's bundle), it is composed with the module held as its
+			// node-engine holds it, and named so; a node that refuses then too is the change's refusal.
+			how := "composed from the change's manifest without its build: "
+			if !m.Composes && m.NotJudged == "" {
+				if again := machine(heldAgain(in, held, awaited), node, depth, r.Declarations); again.Composes {
+					m, how = again, "composed with it held as its node-engine holds it, since it does not compose without its build: "
+				}
+			}
+			words = how + words
+			if m.Composes {
+				m = Machine{NotJudged: words, Declared: m.Declared}
+			} else if m.NotJudged == "" {
+				m.Reason += "; " + words
+			}
+		}
+		cv.Machines[node] = m
 	}
 	return cv, nil
 }
 
+// overrule lifts the hold of the input's node-engine on each module whose manifest the case carries and judges itself
+// (novox/hq ADR 0578 §2): a module the input holds (Refused: the running indexer's verdict, or the running node-engine's
+// decoder), whose bytes this composer reads whole, and that the case gives an accepted verdict or none (a reader takes a
+// manifest with no verdict as accepted, ADR 0578 §3; the running checker drops the verdicts of the change's manifests,
+// issue 1921). It is then no longer Refused nor held in how the node composes. A case's own refusal and bytes this
+// composer does not decode stay held. awaited is each module lifted with no build the input carries: the words for the verdict, or
+// "" where it is composed at its build.
+func overrule(base compose.Input, c Case, reads map[string]ManifestRead, shelf map[string]catalogue.Manifest) (compose.Input, map[string]string) {
+	awaited := map[string]string{}
+	for name, why := range base.Refused {
+		hold := base.Holds[name]
+		if _, carried := c.Manifests[name]; !carried || !reads[name].Accepted || shelf[name].ReadNoFurther() {
+			continue
+		}
+		held := "held by the running node-engine"
+		switch hold.By {
+		case compose.HeldByVerdict:
+			held = "refused only by the running indexer"
+		case compose.HeldByDecoder:
+			held = "not read by the running node-engine's decoder"
+		}
+		judged := "accepted by the check's own validator"
+		if _, given := c.Verdicts[name]; !given {
+			judged = "given no verdict by the check, so taken as accepted (novox/hq ADR 0578 §3)"
+		}
+		awaited[name] = ""
+		if hold.Built == nil && compose.NamesBuild(shelf[name]) {
+			build := "its build"
+			if hold.Build != "" {
+				build += " " + hold.Build
+			}
+			awaited[name] = fmt.Sprintf("%s is %s (%s) and %s, and %s is awaited", name, held, why, judged, build)
+		}
+	}
+	if len(awaited) == 0 {
+		return base, awaited
+	}
+	base.Refused = maps.Clone(base.Refused)
+	for name := range awaited {
+		delete(base.Refused, name)
+	}
+	if how := base.ComposedHow; how != nil {
+		copied := *how
+		copied.Held = maps.Clone(how.Held)
+		for name := range awaited {
+			delete(copied.Held, name)
+		}
+		base.ComposedHow = &copied
+	}
+	return base, awaited
+}
+
+// heldAgain is in with each module overrule lifted and gave no build held again as held, the input before overrule,
+// holds it: its stub, Refused and how the node composes.
+func heldAgain(in, held compose.Input, awaited map[string]string) compose.Input {
+	in.Catalogue = maps.Clone(in.Catalogue)
+	in.Refused = maps.Clone(in.Refused)
+	if in.Refused == nil {
+		in.Refused = map[string]string{}
+	}
+	if how := in.ComposedHow; how != nil {
+		copied := *how
+		copied.Held = maps.Clone(how.Held)
+		in.ComposedHow = &copied
+	}
+	for name, words := range awaited {
+		if words == "" {
+			continue
+		}
+		if stub, ok := held.Catalogue[name]; ok {
+			in.Catalogue[name] = stub
+		} else {
+			in.Catalogue[name] = compose.Stub(in.Catalogue[name])
+		}
+		in.Refused[name] = held.Refused[name]
+		if in.ComposedHow != nil && held.ComposedHow != nil {
+			if why, was := held.ComposedHow.Held[name]; was {
+				if in.ComposedHow.Held == nil {
+					in.ComposedHow.Held = map[string]string{}
+				}
+				in.ComposedHow.Held[name] = why
+			}
+		}
+	}
+	return in
+}
+
+// awaitedOn is the words of each module placed on node that is composed without its build (overrule), joined, sorted.
+func awaitedOn(in compose.Input, node string, awaited map[string]string) string {
+	var out []string
+	for _, n := range in.Nodes {
+		if n.Name != node {
+			continue
+		}
+		for _, m := range n.Assigned {
+			if words := awaited[m]; words != "" {
+				out = append(out, words)
+			}
+		}
+	}
+	sort.Strings(out)
+	return strings.Join(out, "; ")
+}
+
 // HoldingsOf is every seat an assigned module claims, held on the node it is assigned to, in node order: the holdings
 // of an input that has none of record (the assigner's, mesh-delivery-assigner internal/assign input).
 func HoldingsOf(shelf map[string]catalogue.Manifest, nodes []compose.Node) []catalogue.Held {

internal/composer/composer_test.go

@@ -188,7 +188,8 @@ func TestARequestItCannotRunIsRefused(t *testing.T) {
 	})
 	// Issue 774 (the group check of feat/l3-final 278aa5f0ccbe): a tool runner its node-engine holds by its manifest's
 	// verdict is sent as its stub, and the input says it holds it: the sandbox composes the node with it held, never refuses
-	// it for its stub's missing bundle. An input from an engine before the mark is read by Refused alone, the same way.
+	// it for its stub's missing bundle. An input from an engine before the mark is read by Refused alone, the same way. The
+	// case gives the runner its refused verdict, as the checker does for a manifest the change does not bring.
 	t.Run("a refused tool runner the engine sends as its stub", func(t *testing.T) {
 		raw := `{"module": "runner", "version": "1", "tool-runner": true, ` +
 			`"build": {"artifacts": [{"name": "runtime", "kind": "bundle", "language": "go", "from": "cmd/runner", "binary": "runner"}]}}`
@@ -211,7 +212,9 @@ func TestARequestItCannotRunIsRefused(t *testing.T) {
 		for name, sent := range map[string]string{"with the mark": string(wire), "from an engine before it": older} {
 			var r Request
 			r = Request{V: Version, Machines: []string{"a"}, Depth: DepthCompose,
-				Cases: []Case{{Name: "change", Manifests: map[string]Manifest{"runner": {Raw: json.RawMessage(raw)}}}}}
+				Cases: []Case{{Name: "change", Manifests: map[string]Manifest{"runner": {Raw: json.RawMessage(raw)}},
+					Verdicts: bucketstest.Raw(map[string]meshcatalog.VerdictsByDigestValue{"runner": {V: 1, Module: "runner",
+						State: meshcatalogclient.VerdictRefused, Problems: []string{"p"}}})}}}
 			if err := json.Unmarshal([]byte(sent), &r.Input); err != nil {
 				t.Fatal(err)
 			}
@@ -227,6 +230,77 @@ func TestARequestItCannotRunIsRefused(t *testing.T) {
 			}
 		}
 	})
+	// ADR 0578 §2, §3 (issue 1921): a module its node-engine holds by the running indexer's verdict gives way when the case,
+	// judging the change's manifest itself, gives it an accepted verdict or none (the running checker drops the change's
+	// own), and this composer reads its bytes whole. It is composed at the build the input carries for it (Input.Holds);
+	// without one it is composed from the case's manifest and the node is not judged, naming it (never a silent pass), or,
+	// where it cannot compose without its build (a tool runner's bundle), composed held as its node-engine holds it, named
+	// so. Bytes that do not decode here, and a case's own refusal, stay held.
+	t.Run("a held module the case judges itself", func(t *testing.T) {
+		app := `{"module": "app", "version": "1", "resources": [{"id": "server", "type": "container", "name": "app-server", "artifact": "server"}], ` +
+			`"build": {"artifacts": [{"name": "server", "kind": "upstream", "from": "nats:2.11@sha256:` + strings.Repeat("a", 64) + `"}]}}`
+		// A module that names a build and composes from its manifest without it.
+		plain := `{"module": "app", "version": "1", "resources": [{"id": "d", "type": "directory", "path": "/var/lib/app"}], ` +
+			`"build": {"artifacts": [{"name": "server", "kind": "upstream", "from": "nats:2.11@sha256:` + strings.Repeat("a", 64) + `"}]}}`
+		runner := `{"module": "app", "version": "1", "tool-runner": true, ` +
+			`"build": {"artifacts": [{"name": "runtime", "kind": "bundle", "language": "go", "from": "cmd/runner", "binary": "runner"}]}}`
+		built, err := catalogue.Decode([]byte(`{"module": "app", "version": "1", "resources": [{"id": "built", "type": "directory", "path": "/var/lib/app-built"}]}`))
+		if err != nil {
+			t.Fatal(err)
+		}
+		verdict := func(state string) map[string]json.RawMessage {
+			return bucketstest.Raw(map[string]meshcatalog.VerdictsByDigestValue{"app": {V: 1, Module: "app", State: state, Problems: []string{"p"}}})
+		}
+		const why = "the indexer's judge refused the manifest of app"
+		for _, row := range []struct {
+			name     string
+			raw      string
+			verdicts map[string]json.RawMessage
+			hold     *compose.Hold
+			composes bool   // the machine composes, judged
+			path     string // a path declared on it
+			words    string // in NotJudged, else in the read's refusal
+		}{
+			{"an accepted verdict, its build carried", app, verdict(meshcatalogclient.VerdictAccepted),
+				&compose.Hold{By: compose.HeldByVerdict, Build: "fp1", Built: &built}, true, "/var/lib/app-built", ""},
+			{"no verdict, its build carried", app, nil, &compose.Hold{By: compose.HeldByVerdict, Build: "fp1", Built: &built}, true, "/var/lib/app-built", ""},
+			{"an accepted verdict, its build awaited", plain, verdict(meshcatalogclient.VerdictAccepted), &compose.Hold{By: compose.HeldByVerdict, Build: "fp1"}, false, "",
+				"composed from the change's manifest without its build: app is refused only by the running indexer (" + why + ") and accepted by the check's own validator, and its build fp1 is awaited"},
+			{"no verdict, from an engine before holds", plain, nil, nil, false, "",
+				"app is held by the running node-engine (" + why + ") and given no verdict by the check, so taken as accepted"},
+			{"a container, its image awaited", app, nil, &compose.Hold{By: compose.HeldByVerdict}, false, "",
+				"composed with it held as its node-engine holds it, since it does not compose without its build: app is refused only by the running indexer"},
+			{"a tool runner, its bundle awaited", runner, nil, &compose.Hold{By: compose.HeldByVerdict}, false, "",
+				"composed with it held as its node-engine holds it, since it does not compose without its build: app is refused only by the running indexer"},
+			{"bytes that do not decode", `{"module": "app", "version": "1", "resources": "none"}`, nil, &compose.Hold{By: compose.HeldByVerdict, Built: &built}, true, "", "does not decode"},
+			{"the case's own refusal", app, verdict(meshcatalogclient.VerdictRefused), &compose.Hold{By: compose.HeldByVerdict, Built: &built}, true, "", "refused the manifest of app"},
+		} {
+			man, _ := catalogue.Decode([]byte(app))
+			in := compose.Input{Catalogue: map[string]catalogue.Manifest{"app": compose.Stub(man)}, Refused: map[string]string{"app": why},
+				Nodes: []compose.Node{{Name: "a", Assigned: []string{"app"}}}}
+			if row.hold != nil {
+				in.Holds = map[string]compose.Hold{"app": *row.hold}
+			}
+			v, err := Run(withInput(t, Request{V: Version, Machines: []string{"a"}, Depth: DepthCompose, Declarations: true,
+				Cases: []Case{{Name: "change", Manifests: map[string]Manifest{"app": {Raw: json.RawMessage(row.raw)}}, Verdicts: row.verdicts}}}, in))
+			if err != nil {
+				t.Fatal(err)
+			}
+			m, read := v.Cases[0].Machines["a"], v.Cases[0].Manifests["app"]
+			declared := false
+			for _, p := range m.Declared {
+				declared = declared || p.Path == row.path
+			}
+			switch {
+			case m.Composes != row.composes || (row.path != "" && !declared):
+				t.Errorf("%s: machine %+v", row.name, m)
+			case !row.composes && (m.NotJudged == "" || !strings.Contains(m.NotJudged, row.words)):
+				t.Errorf("%s: not judged %q, want it naming %q", row.name, m.NotJudged, row.words)
+			case row.composes && row.path == "" && (read.Accepted || !strings.Contains(read.Refused, row.words)):
+				t.Errorf("%s: read %+v, want held naming %q", row.name, read, row.words)
+			}
+		}
+	})
 	v, err := Run(Request{V: Version, Cases: []Case{{Name: "only-read", Manifests: map[string]Manifest{"demo": {Raw: json.RawMessage(demo)}}}}})
 	if err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted || v.Cases[0].Machines != nil || v.SDK == "" {
 		t.Fatalf("a request with no machines: %+v, %v", v, err)