Take the indexer's verdict on a manifest instead of judging it, and name a held module (ADR 0578)

Issue 1585, S17: the engine decodes manifests leniently; a module whose manifest the indexer refused is held and named,
its data key stands, and every other module's items are still published. Policy, HoldsData and the certificate check no
longer parse with the judge.

Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL

internal/selfcompose/data.go

@@ -39,8 +39,9 @@ func (itemsReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow,
 	return nil, nil
 }
 
-// dataItemsOf works out the data items of the scope's node (frombus.DataItems); a test sets what it answers.
-var dataItemsOf = frombus.DataItems
+// dataItemsOf works out the data items of the scope's node and, by module, why each module held for a manifest the
+// indexer refused is left out (frombus.DataItemsHeld, ADR 0578 §3); a test sets what it answers.
+var dataItemsOf = frombus.DataItemsHeld
 
 // publishData keeps the data items of this node's modules in the engine's state data, when a pass asked it (dataDue).
 func (c *Composer) publishData(ctx context.Context) {
@@ -57,19 +58,29 @@ func (c *Composer) publishData(ctx context.Context) {
 	if scope.Node == "" {
 		scope.Node = c.env.Node
 	}
-	items, unplaced, err := dataItemsOf(ctx, itemsReader{reader{c: c}}, c.env.Mesh, scope)
+	items, unplaced, held, err := dataItemsOf(ctx, itemsReader{reader{c: c}}, c.env.Mesh, scope)
 	if err != nil {
 		c.sayData("the data items of this node's modules cannot be worked out, and what the engine's state " +
 			delivery.EngineDataBucket + " holds stands; " + c.oweData() + ": " + err.Error())
 		return
 	}
+	// A module held for a refused manifest declares nothing this pass can read: it is named, its other modules' items
+	// are kept as ever, and its own key stands as it was until the manifest is accepted (ADR 0578 §3).
+	if len(held) > 0 {
+		var lines []string
+		for _, m := range heldModules(held) {
+			lines = append(lines, m+": "+held[m])
+		}
+		c.env.Say("data items of " + strings.Join(heldModules(held), ", ") + " not worked out, as the manifest is refused " +
+			"and the module held (its key stands as it was): " + strings.Join(lines, "; "))
+	}
 	if len(unplaced) > 0 {
 		c.env.Say("data items left out as not placeable: " + strings.Join(unplaced, "; "))
 	}
 	c.mu.Lock()
 	digests := c.digestsByModule()
 	c.mu.Unlock()
-	wrote, removed, err := c.writeData(ctx, items, digests)
+	wrote, removed, err := c.writeData(ctx, items, digests, held)
 	if err != nil {
 		c.sayData("the data items of this node's modules are not all kept in " + delivery.EngineDataBucket +
 			"; " + c.oweData() + ": " + err.Error())
@@ -130,7 +141,7 @@ func (c *Composer) digestsByModule() map[string]string {
 // writeData writes each module's key whose value differs from what the key holds, and deletes this node's keys of
 // modules with no items here now. The keys are read once per run (dataKept), so a restart that finds the same items
 // writes nothing.
-func (c *Composer) writeData(ctx context.Context, items map[string][]catalogue.PlacedDataItem, digests map[string]string) (wrote, removed int, err error) {
+func (c *Composer) writeData(ctx context.Context, items map[string][]catalogue.PlacedDataItem, digests map[string]string, held map[string]string) (wrote, removed int, err error) {
 	store := c.env.Data
 	if c.dataKept == nil {
 		pattern, err := delivery.EngineDataNodeKeys(c.env.Node)
@@ -198,7 +209,7 @@ func (c *Composer) writeData(ctx context.Context, items map[string][]catalogue.P
 	}
 	var gone []string
 	for key := range c.dataKept {
-		if _, still := want[key]; !still {
+		if _, still := want[key]; !still && !heldKey(c.env.Node, key, held) {
 			gone = append(gone, key)
 		}
 	}
@@ -216,3 +227,13 @@ func (c *Composer) writeData(ctx context.Context, items map[string][]catalogue.P
 	}
 	return wrote, removed, nil
 }
+
+// heldKey is whether key is the key of a module held for a refused manifest, which stands as it was.
+func heldKey(node, key string, held map[string]string) bool {
+	for module := range held {
+		if k, err := delivery.EngineDataKey(node, module); err == nil && k == key {
+			return true
+		}
+	}
+	return false
+}