mesh-sdk/go/compose/composed.go

package compose

import (
	"encoding/json"
	"errors"
	"fmt"
	"slices"
	"sort"
	"strings"

	"git.novox.be/novox/mesh-sdk/go/catalogue"
	"git.novox.be/novox/mesh-sdk/go/declaration"
	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/overlay"
)

// Compositions is one node's compositions (novox/hq ADR 0539 clauses 1 and 2): its declaration composed over in, as
// the controller composes it, split into each module's composition at the fingerprint fps gives it and the node's own
// (`<node>.node`), by key (delivery.ComposedKey). Each resource says its module, whatever the node reported it reads.
// A module fps gives no fingerprint for is not composed and is named in skipped, so a caller composing one module
// (mesh-hand-compose) gets that one; but for a module the node runs inside its own composition (Node.InOwn, and the
// private network's always), which fps names at no fingerprint of the new form: its resources go into the node's own,
// their module taken off and their ids still naming it, and its filled paths with them, as the controller's composed
// writer folds them (issue 1126). Each module's Data says whether its manifest declares data. The compositions are not
// signed and not yet by digest: the writer does both (delivery.SignComposed, Composed.ByDigest).
func Compositions(in Input, node string, fps map[string]string, mode Mode) (out map[string]delivery.Composed, skipped []string, err error) {
	out, skipped, _, err = compositions(in, node, fps, mode, false)
	return out, skipped, err
}

// ErrStoodIn is a composition that holds a stand-in for a value the input does not carry (an own secret, a grant's
// sealed copy, a vault box, the certificate, the bus's user list) or a port it does not give: what a send would make,
// never what may be applied.
var ErrStoodIn = errors.New("compose: the composition stands in for what its input does not carry")

// BusUsersStandIn is what a composition says it stood in for when its input carries no bus user list.
const BusUsersStandIn = "the bus's user list"

// CompositionsApplied is Compositions for a node that applies what it composed itself (novox/hq ADR 0547 §2, issue
// 1126): a declaration that stands in for anything (Declared.StoodIn, Foreseen, ForeseenPorts) is refused with
// ErrStoodIn, naming each, so a stand-in is never applied as if it were the value; but for the bus's user list
// (BusUsersStandIn), which the bus node holds out of every apply of a composition it made itself.
func CompositionsApplied(in Input, node string, fps map[string]string) (map[string]delivery.Composed, []string, error) {
	out, skipped, _, err := compositions(in, node, fps, Reading, true)
	return out, skipped, err
}

// CompositionsShadow is CompositionsApplied for a composition that is compared and never applied (a node's answer to a
// placement candidate, novox/hq ADR 0559 §4): composed in the Foreseeing mode, so an own secret never issued (a module
// the candidate adds) is a named stand-in rather than an error, and what it stands in for is answered (stood, sorted,
// the bus's user list left out) rather than refused, so the caller judges each stand-in: a value not yet issued for a
// module the candidate adds, or one missing for a module the node runs. A module whose every value is held composes
// as CompositionsApplied composes it: the modes differ only in an own secret never issued.
func CompositionsShadow(in Input, node string, fps map[string]string) (out map[string]delivery.Composed, skipped, stood []string, err error) {
	return compositions(in, node, fps, Foreseeing, false)
}

// CompositionsIn is the node's compositions in mode with what they stand in for answered (stood, sorted, the bus's user
// list left out) rather than refused: CompositionsShadow in Foreseeing, and in Reading what CompositionsApplied refuses
// before it is refused, so a caller that applies refuses each stand-in to the module it is for alone (issue 1126).
func CompositionsIn(in Input, node string, fps map[string]string, mode Mode) (out map[string]delivery.Composed, skipped, stood []string, err error) {
	return compositions(in, node, fps, mode, false)
}

func compositions(in Input, node string, fps map[string]string, mode Mode, applied bool) (out map[string]delivery.Composed, skipped, stood []string, err error) {
	reading := in
	reading.Nodes = append([]Node(nil), in.Nodes...)
	found := false
	inOwn := map[string]bool{}
	held := map[string]bool{}
	for i := range reading.Nodes {
		if reading.Nodes[i].Name == node {
			reading.Nodes[i].ReadsModules = true
			found = true
			for m, ok := range reading.Nodes[i].InOwn {
				inOwn[m] = ok
			}
			for m, ok := range reading.Nodes[i].Held {
				held[m] = ok
			}
		}
	}
	// The private network's module is built by nobody: no fingerprint of the new form ever names it.
	inOwn[overlay.Name] = true
	folds := func(module string) bool {
		return module != "" && inOwn[module] && !strings.HasPrefix(fps[module], delivery.SourcePrefix2)
	}
	if !found {
		return nil, nil, nil, fmt.Errorf("compose: the mesh has no node %q", node)
	}
	// A folded module built by a build of the old form (Node.RunsOwn) is left to the node: composed from what runs of it
	// (Carried), so what it provides and requires still shapes the rest and what the others read of it (its published
	// ports, the paths it owns) is what runs, and its resources are then the ones the node runs.
	carried := map[string]bool{}
	var runsOwn *delivery.Composed
	for i := range reading.Nodes {
		n := &reading.Nodes[i]
		if n.Name != node || n.RunsOwn == nil {
			continue
		}
		runsOwn = n.RunsOwn
		candidates := map[string]bool{}
		for m := range inOwn {
			candidates[m] = true
		}
		for m := range held {
			candidates[m] = true
		}
		for m := range candidates {
			// A held module is carried as it runs wherever the node's own runs it, folded or leaving the fold for a
			// composition of its own (its want new, its names still unset): its stub is never what the node's own applies,
			// and what the node's own applies never drops what runs of it (issue 1126, review of the apply; issue 1945).
			heldHere := held[m] && RunsAnyOf(runsOwn, m)
			if !folds(m) && !heldHere {
				continue
			}
			man, ok := n.Kept[m]
			if !ok {
				man, ok = reading.Catalogue[m]
			}
			if !ok || (!NamesBuild(man) && !held[m]) {
				continue
			}
			carried[m] = true
			kept := make(map[string]catalogue.Manifest, len(n.Kept)+1)
			for k, v := range n.Kept {
				kept[k] = v
			}
			kept[m] = Carried(man, m, runsOwn)
			n.Kept = kept
		}
	}
	isCarried := func(module, id string) bool {
		if module == "" {
			module, _ = declaration.ModuleGuess(id)
		}
		return carried[module]
	}
	d, problems, err := ComposeIn(reading, node, mode)
	if err != nil {
		return nil, nil, nil, err
	}
	if len(problems) > 0 {
		return nil, nil, nil, fmt.Errorf("compose: %s's declaration is refused by the node-engine's validator: %v", node, problems)
	}
	for _, x := range d.StoodIn {
		// The bus's user list is the bus node's own to follow, written alone and never from a composition the node
		// made itself (ADR 0547 §7, §8): its stand-in is held out of the apply, not applied.
		if x != BusUsersStandIn {
			stood = append(stood, x)
		}
	}
	stood = append(stood, d.Foreseen...)
	for _, p := range d.ForeseenPorts {
		stood = append(stood, "port "+p)
	}
	sort.Strings(stood)
	stood = slices.Compact(stood) // an own secret is both stood in for and foreseen
	if applied && len(stood) > 0 {
		return nil, nil, nil, fmt.Errorf("%w on %s: %s", ErrStoodIn, node, strings.Join(stood, ", "))
	}
	body, err := d.Body()
	if err != nil {
		return nil, nil, nil, err
	}
	var envelope struct {
		Resources   []json.RawMessage     `json:"resources"`
		Adoption    *declaration.Adoption `json:"adoption"`
		Filled      []string              `json:"filled_files"`
		Provisioner string                `json:"provisioner_account"`
	}
	if err := json.Unmarshal(body, &envelope); err != nil {
		return nil, nil, nil, err
	}
	var kept []json.RawMessage
	keptPaths := map[string]bool{}
	foldedPaths := map[string]bool{}
	left := map[string]bool{}
	for _, raw := range envelope.Resources {
		var head struct {
			ID     string `json:"id"`
			Module string `json:"module"`
			Type   string `json:"type"`
			Path   string `json:"path"`
		}
		if err := json.Unmarshal(raw, &head); err != nil {
			return nil, nil, nil, err
		}
		if isCarried(head.Module, head.ID) {
			continue // what the stub composed of it: the node's own carries what runs of it instead
		}
		if folds(head.Module) {
			// Into the node's own, in its place among the node's resources, its module taken off (mesh-controller's
			// compositionsOf): the node's own composition is then the controller's, byte for byte.
			var fields map[string]json.RawMessage
			if err := json.Unmarshal(raw, &fields); err != nil {
				return nil, nil, nil, err
			}
			delete(fields, "module")
			if raw, err = json.Marshal(fields); err != nil {
				return nil, nil, nil, err
			}
			if head.Type == string(declaration.TypeFile) {
				foldedPaths[head.Path] = true
			}
			kept = append(kept, raw)
			continue
		}
		if _, ok := fps[head.Module]; head.Module != "" && !ok {
			left[head.Module] = true
			continue
		}
		if head.Type == string(declaration.TypeFile) {
			keptPaths[head.Path] = true
		}
		kept = append(kept, raw)
	}
	// What runs of each carried module, in the node's own composition, each after the resource it runs after.
	if len(carried) > 0 {
		var err error
		if kept, foldedPaths, err = carry(kept, foldedPaths, runsOwn, isCarried); err != nil {
			return nil, nil, nil, err
		}
		for _, p := range runsOwn.Filled {
			if foldedPaths[p] && !slices.Contains(envelope.Filled, p) {
				envelope.Filled = append(envelope.Filled, p)
			}
		}
	}
	// A filled file of a module's own composition goes with that module (SplitComposed); one of a folded module goes with
	// the node's own, after them, in the declaration's order; one of a module left out is dropped.
	var filled, ownFilled []string
	for _, p := range envelope.Filled {
		switch {
		case foldedPaths[p]:
			ownFilled = append(ownFilled, p)
		case keptPaths[p]:
			filled = append(filled, p)
		}
	}
	out, err = delivery.SplitComposed(node, kept, fps, filled,
		delivery.NodeOwn{Adoption: envelope.Adoption, Provisioner: envelope.Provisioner})
	if err != nil {
		return nil, nil, nil, err
	}
	if len(ownFilled) > 0 {
		own, err := delivery.ComposedNodeKey(node)
		if err != nil {
			return nil, nil, nil, err
		}
		c := out[own]
		c.Filled = append(c.Filled, ownFilled...)
		out[own] = c
	}
	for key, c := range out {
		if held[c.Module] {
			delete(out, key) // its stub's: the node keeps what it last applied of it (Node.Held)
			continue
		}
		if m, ok := in.Catalogue[c.Module]; ok && c.Module != "" {
			c.Data = len(m.DataItems()) > 0
			out[key] = c
		}
	}
	for m := range left {
		skipped = append(skipped, m)
	}
	sort.Strings(skipped)
	return out, skipped, stood, nil
}

// RunsAnyOf says runs (a composition as it runs) holds a resource of module, by its module or the module its id names.
func RunsAnyOf(runs *delivery.Composed, module string) bool {
	if runs == nil {
		return false
	}
	for _, raw := range runs.Resources {
		var head struct {
			ID     string `json:"id"`
			Module string `json:"module"`
		}
		if json.Unmarshal(raw, &head) != nil {
			continue
		}
		owner := head.Module
		if owner == "" {
			owner, _ = declaration.ModuleGuess(head.ID)
		}
		if owner == module {
			return true
		}
	}
	return false
}

// NamesBuild says a manifest names what a build makes: its build, or a resource naming an artifact. A node cannot
// compose such a module folded into its own composition: the build of the old form it runs is held by the controller
// alone (issue 1126).
func NamesBuild(m catalogue.Manifest) bool {
	if m.Build != nil {
		return true
	}
	for _, r := range m.Resources {
		if _, ok := r["artifact"]; ok {
			return true
		}
	}
	return false
}

// Stub is m with nothing to build and no resource but its directories: what it provides, requires, serves and grants
// still shape the rest of the composition, the places its own secrets, data and grants name (${dir:…}) still resolve,
// and nothing else of its own is composed (issue 1126: nats's ${dir:mesh-state} on novox). A module refused in a
// node's answer is composed so; a folded one the node runs is Carried.
func Stub(m catalogue.Manifest) catalogue.Manifest {
	// The paths it owns stay its manifest's, as the controller tells the holder of node-files them (issue 1126, seed-8).
	if m.OwnsPaths == nil {
		m.OwnsPaths = catalogue.DeclaredPaths(m)
	}
	m.Build = nil
	var dirs []map[string]any
	for _, r := range m.Resources {
		if r["type"] == "directory" {
			dirs = append(dirs, r)
		}
	}
	m.Resources = dirs
	return m
}

// Carried is m, folded into its node's own composition at a build of the old form, as the node runs it (issue 1126,
// C5): nothing to build; its containers and directories those the node's own composition (runs) carries of it, their
// module's word taken off their ids, with the manifest's directories it does not run. So what other modules read of it
// is what runs: the machine ports its containers publish (the filter's rules, the routes reaching it: MachineSide). The
// paths it owns, which the holder of node-files is told, are its manifest's, as for its Stub: the controller composes
// them from the manifest, not from every path a resource of it runs at (issue 1126, seed-8: f1-circuits, snake and
// txt-game on ace, whose manifests leave their directories to the mesh). Its other resources are not taken, since the
// composition makes some of them again from its manifest (its secrets' files) and one id twice is refused; none of them
// is composed into what is applied either way (compositions leaves a carried module's out, and carries the ones that
// run). What it provides, requires, listens on and grants is its manifest's. A module whose resources runs holds none
// of is its Stub, as before.
func Carried(m catalogue.Manifest, module string, runs *delivery.Composed) catalogue.Manifest {
	if runs == nil {
		return Stub(m)
	}
	var taken []map[string]any
	runsAny := false
	ids := map[string]bool{}
	for _, raw := range runs.Resources {
		var r map[string]any
		if json.Unmarshal(raw, &r) != nil {
			continue
		}
		id, _ := r["id"].(string)
		owner, _ := r["module"].(string)
		if owner == "" {
			owner, _ = declaration.ModuleGuess(id)
		}
		if owner != module {
			continue
		}
		runsAny = true
		if typ := r["type"]; typ != "container" && typ != "directory" {
			continue
		}
		r["id"] = strings.TrimPrefix(id, module+".")
		delete(r, "module")
		ids[r["id"].(string)] = true
		taken = append(taken, r)
	}
	if !runsAny {
		return Stub(m)
	}
	stub := Stub(m)
	for _, r := range stub.Resources {
		if id, _ := r["id"].(string); !ids[id] {
			taken = append(taken, r)
		}
	}
	stub.Resources = taken
	return stub
}

// carry puts into the node's resources (kept, in the declaration's order) the resources of the node's own composition as
// it runs (runs) that belong to a carried module, each after the resource it runs after, or first when that one is
// not among them; a file among them is a folded path.
func carry(kept []json.RawMessage, foldedPaths map[string]bool, runs *delivery.Composed, carried func(module, id string) bool) ([]json.RawMessage, map[string]bool, error) {
	idOf := func(raw json.RawMessage) (string, string, string, string, error) {
		var head struct {
			ID     string `json:"id"`
			Module string `json:"module"`
			Type   string `json:"type"`
			Path   string `json:"path"`
		}
		err := json.Unmarshal(raw, &head)
		return head.ID, head.Module, head.Type, head.Path, err
	}
	present := map[string]bool{}
	for _, raw := range kept {
		if id, module, _, _, err := idOf(raw); err == nil && module == "" {
			present[id] = true
		}
	}
	// By the id of the resource of the node's own it runs after, "" for first: the last one before it that is composed.
	after := map[string][]json.RawMessage{}
	anchor := ""
	for _, raw := range runs.Resources {
		id, module, typ, path, err := idOf(raw)
		if err != nil {
			return nil, nil, fmt.Errorf("compose: a resource the node runs in its own composition: %w", err)
		}
		if !carried(module, id) {
			if module == "" && present[id] {
				anchor = id
			}
			continue
		}
		after[anchor] = append(after[anchor], raw)
		if typ == string(declaration.TypeFile) {
			foldedPaths[path] = true
		}
	}
	out := make([]json.RawMessage, 0, len(kept)+len(runs.Resources))
	out = append(out, after[""]...)
	for _, raw := range kept {
		out = append(out, raw)
		if id, module, _, _, err := idOf(raw); err == nil && module == "" {
			out = append(out, after[id]...)
		}
	}
	return out, foldedPaths, nil
}