Let a requirement be optional, binding only a provider that already runs (hq issue 166, ADR 0330)

Making every soft dependency a provision made it hard: a mesh without influxdb could not assign
grafana, and one with influxdb in its catalogue would install it for grafana. A requirement may now
be written {"name": ..., "optional": true}. It resolves as a hard one when a provider is assigned or
reachable, and is otherwise absent: never refused, nothing installed for it. Its binding, credential
and only-when-bound files are left out, its ${bound:...|default:...} placeholders take their
defaults, the module check refuses one that could render empty, and plan and node list it unbound.

internal/catalogue/optional_requirements.go

@@ -0,0 +1,257 @@
+package catalogue
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+)
+
+// A requirement a module can run without (novox/hq ADR 0330, issue 166).
+//
+// **Hard stays the default.** A requirement written as a bare name is hard, exactly as before: it
+// resolves, or the assignment is refused, and a single catalogue candidate is installed for it. One
+// written `{"name": "influxdb-api", "optional": true}` resolves the same way when a provider is
+// already assigned in the mesh or reachable from the consumer (§2), and is otherwise **absent**: not
+// refused, and nothing is installed for it, on this machine or elsewhere (§3).
+//
+// **An absent requirement is never rendered as an empty value** (§4–§6). Its binding, its pair
+// credential and its grant are not composed, so a file that names one of them is delivered only while
+// the requirement is bound (`only-when-bound`), and a `${bound:…}` placeholder of it outside such a
+// file names the value it takes while absent: `${bound:<provision>:<key>|default:<value>}`. The
+// module check refuses anything else, so the gap is found when the manifest is written, not on a
+// machine where it reads as a blank.
+
+// OnlyWhenBound is a resource's field naming the optional requirement it is delivered only while
+// bound (ADR 0330 §5). Said in the catalogue and dropped before the declaration is sent: the
+// node-engine parses strictly and knows no such field.
+const OnlyWhenBound = "only-when-bound"
+
+// requirementForm is one entry of `requires` in its long form.
+type requirementForm struct {
+	Name     string `json:"name"`
+	Optional bool   `json:"optional,omitempty"`
+}
+
+// readRequires reads `requires` in both of its forms — a bare name, or {name, optional} — into the
+// names and which of them are optional.
+func readRequires(raw json.RawMessage) ([]string, map[string]bool, error) {
+	var entries []json.RawMessage
+	if err := json.Unmarshal(raw, &entries); err != nil {
+		return nil, nil, fmt.Errorf("requires: a list of names, or of {name, optional}: %w", err)
+	}
+	var names []string
+	optional := map[string]bool{}
+	for i, e := range entries {
+		var plain string
+		if err := json.Unmarshal(e, &plain); err == nil {
+			names = append(names, plain)
+			continue
+		}
+		var full requirementForm
+		dec := json.NewDecoder(bytes.NewReader(e))
+		dec.DisallowUnknownFields()
+		if err := dec.Decode(&full); err != nil {
+			return nil, nil, fmt.Errorf("requires[%d]: a name, or {name, optional} (novox/hq ADR 0330): %w", i, err)
+		}
+		if strings.TrimSpace(full.Name) == "" {
+			return nil, nil, fmt.Errorf("requires[%d] names no requirement: {name, optional}", i)
+		}
+		names = append(names, full.Name)
+		if full.Optional {
+			optional[full.Name] = true
+		}
+	}
+	if len(optional) == 0 {
+		optional = nil
+	}
+	return names, optional, nil
+}
+
+// writeRequires is `requires` in the form it was read: a bare name for a hard requirement, the long
+// form for an optional one, so a manifest that went through the mesh comes out as it went in.
+func writeRequires(m Manifest) (json.RawMessage, error) {
+	out := make([]any, 0, len(m.Requires))
+	for _, r := range m.Requires {
+		if m.OptionalRequires[r] {
+			out = append(out, requirementForm{Name: r, Optional: true})
+			continue
+		}
+		out = append(out, r)
+	}
+	return json.Marshal(out)
+}
+
+// IsOptional is whether this module requires the name optionally (ADR 0330 §1). A name it contributes
+// to without requiring, or requires as a bare name, is hard.
+func (m Manifest) IsOptional(requirement string) bool { return m.OptionalRequires[requirement] }
+
+// wantsHard is whether this module wants the name as a hard requirement: required without
+// `optional`, or contributed to.
+func (m Manifest) wantsHard(name string) bool {
+	for _, w := range m.Wants() {
+		if w == name {
+			return !m.IsOptional(name)
+		}
+	}
+	return false
+}
+
+// Absent is one optional requirement of one consumer that nothing assigned answers (ADR 0330 §3):
+// left unbound rather than refused, and listed by `plan` and `node` (§7).
+type Absent struct {
+	// Consumer is the module that requires it optionally.
+	Consumer string `json:"consumer"`
+	// Requirement is what it requires.
+	Requirement string `json:"requirement"`
+}
+
+// AbsentFor is the optional requirements of one module this resolution leaves unbound.
+func (r Resolution) AbsentFor(module string) map[string]bool {
+	out := map[string]bool{}
+	for _, a := range r.Absent {
+		if a.Consumer == module {
+			out[a.Requirement] = true
+		}
+	}
+	return out
+}
+
+// absentOf is, for each requirement left absent, every module of the set that requires it
+// optionally, sorted by consumer then requirement.
+func absentOf(names map[string]bool, order []string, catalogue map[string]Manifest) []Absent {
+	var out []Absent
+	for _, module := range order {
+		m := catalogue[module]
+		for _, r := range m.Requires {
+			if names[r] && m.IsOptional(r) {
+				out = append(out, Absent{Consumer: module, Requirement: r})
+			}
+		}
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Consumer != out[j].Consumer {
+			return out[i].Consumer < out[j].Consumer
+		}
+		return out[i].Requirement < out[j].Requirement
+	})
+	return out
+}
+
+// withoutAbsent drops the resources delivered only while a requirement is bound whose requirement is
+// absent here (ADR 0330 §5), and the field itself from every resource that keeps it.
+func withoutAbsent(resources []map[string]any, absent map[string]bool) []map[string]any {
+	out := make([]map[string]any, 0, len(resources))
+	for _, r := range resources {
+		when, has := r[OnlyWhenBound]
+		if !has {
+			out = append(out, r)
+			continue
+		}
+		if absent[fmt.Sprint(when)] {
+			continue
+		}
+		kept := make(map[string]any, len(r))
+		for k, v := range r {
+			if k != OnlyWhenBound {
+				kept[k] = v
+			}
+		}
+		out = append(out, kept)
+	}
+	return out
+}
+
+// secretNamesOf is every `${secret:…}` name that carries a credential of this requirement: its own
+// name, and each local name it keeps several under (ADR 0094).
+func secretNamesOf(m Manifest, requirement string) []string {
+	names := []string{requirement}
+	for local := range m.SecretsMany[requirement] {
+		names = append(names, local)
+	}
+	sort.Strings(names)
+	return names
+}
+
+// OptionalProblems is what the module check refuses of a manifest's optional requirements and its
+// only-when-bound resources (ADR 0330 §4–§6).
+func OptionalProblems(m Manifest) []string {
+	var problems []string
+	required := map[string]bool{}
+	for _, r := range m.Requires {
+		if required[r] && m.IsOptional(r) {
+			problems = append(problems, fmt.Sprintf("%s requires %q twice; say it once, optional or not", m.Module, r))
+		}
+		required[r] = true
+	}
+	// A contribution is to whoever answers the requirement, so it goes with it: contributing to one
+	// that is optional is allowed, and composed only while it is bound.
+	for _, r := range m.Resources {
+		id := fmt.Sprint(r["id"])
+		when, marked := r[OnlyWhenBound]
+		on := ""
+		if marked {
+			name, ok := when.(string)
+			switch {
+			case !ok || name == "":
+				problems = append(problems, fmt.Sprintf(
+					"%s's resource %s says %s %v; it names one requirement the module requires optionally",
+					m.Module, id, OnlyWhenBound, when))
+				continue
+			case fmt.Sprint(r["type"]) != "file":
+				problems = append(problems, fmt.Sprintf(
+					"%s's resource %s is a %v and says %s: only a file is left out while a requirement is "+
+						"absent (novox/hq ADR 0330 §5)", m.Module, id, r["type"], OnlyWhenBound))
+				continue
+			case !required[name]:
+				problems = append(problems, fmt.Sprintf(
+					"%s's resource %s is delivered only when %q is bound, and %s does not require it",
+					m.Module, id, name, m.Module))
+				continue
+			case !m.IsOptional(name):
+				problems = append(problems, fmt.Sprintf(
+					"%s's resource %s is delivered only when %q is bound, which is a hard requirement and "+
+						"always bound: mark the requirement optional, or drop %s", m.Module, id, name, OnlyWhenBound))
+				continue
+			}
+			on = name
+		}
+		if fmt.Sprint(r["type"]) != "file" {
+			continue
+		}
+		content, _ := r["content"].(string)
+		for _, use := range bound.FindAllStringSubmatch(content, -1) {
+			provision, key, hasDefault := use[1], use[2], use[3] != ""
+			if !m.IsOptional(provision) {
+				if hasDefault {
+					problems = append(problems, fmt.Sprintf(
+						"%s's resource %s gives ${bound:%s:%s} a default, and %q is a hard requirement, always "+
+							"bound: a default it never renders only misleads a reader", m.Module, id, provision, key, provision))
+				}
+				continue
+			}
+			if !hasDefault && on != provision {
+				problems = append(problems, fmt.Sprintf(
+					"%s's resource %s says ${bound:%s:%s}, and %q is optional: while it is absent this would "+
+						"be empty. Name its value then — ${bound:%s:%s|default:<value>} — or deliver the file "+
+						"only while it is bound, with \"%s\": %q (novox/hq ADR 0330 §4)",
+					m.Module, id, provision, key, provision, provision, key, OnlyWhenBound, provision))
+			}
+		}
+		for _, provision := range m.Requires {
+			if !m.IsOptional(provision) || on == provision {
+				continue
+			}
+			for _, name := range secretNamesOf(m, provision) {
+				if strings.Contains(content, "${secret:"+name+"}") {
+					problems = append(problems, fmt.Sprintf(
+						"%s's resource %s says ${secret:%s}, the credential of %q, which is optional and not "+
+							"delivered while absent: deliver the file only while it is bound, with \"%s\": %q "+
+							"(novox/hq ADR 0330 §6)", m.Module, id, name, provision, OnlyWhenBound, provision))
+				}
+			}
+		}
+	}
+	return problems
+}