mesh-delivery/modules/mesh-delivery-checker/internal/check/engines.go

package check

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-sdk/go/composer"
	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// **A check's composition is the checker's, made from what each node's engine answers** (novox/hq ADR 0593 §7, §9).
// Only a node knows whether a change composes there: the checker asks every placed node's engine
// `node-engine.compose-candidate` in parallel, with the check's candidate, and waits at most its answer bound
// (compose-answer-bound, 60 s by default). A node that refuses fails the check, naming the node and each module with the
// engine's words; a node that does not answer within the bound, answers unready or cannot be asked is named, "not composed
// on that node: no answer", and the check passes with that warning: its own composition judges again when it applies,
// and the canary rule holds. The checker decides by the answer's verdict, never by its words.
//
// **A group that changes the node-engine's composition code** (a member of the node-engine's repository) is not judged by
// the running engines, which cannot judge their own next code: each node's engine answers the composition input it
// composed from (ComposeCandidateAsk.Input), and the composer program built from that member's head composes it in a
// build module's sandbox (Sandbox), with every manifest of the catalogue as the case leaves it and the indexer's verdicts.

// EngineAsker asks one node's engine to compose a check's candidate (the verb node-engine.compose-candidate on that node).
type EngineAsker interface {
	ComposeCandidate(ctx context.Context, node string, ask delivery.ComposeCandidateAsk) (delivery.ComposeCandidateAnswer, error)
}

// Sandbox runs a compose job on the Go build seat and answers the composer's verdict.
type Sandbox interface {
	Compose(ctx context.Context, job delivery.ComposeJob) (composer.Verdict, error)
}

// EngineCompositions is the checker's composition of a check (Examiner.Compositions).
type EngineCompositions struct {
	Asker EngineAsker
	// Placed names every node in placement now (the seat mesh-placement's state), or why it cannot be read: then no node
	// can be asked, and the composition is not made (an error), never passed with nobody asked.
	Placed func() ([]string, error)
	// Bound is the answer bound (the setting compose-answer-bound); zero is delivery.ComposeAnswerBound.
	Bound time.Duration
	// Composer is the commit of the node-engine the mesh runs, named as what composed (ADR 0548 §6's running commit).
	Composer func() string
	// Sandbox runs the composer of a node-engine member's head; nil leaves such a group judged by the running engines,
	// said in a warning.
	Sandbox Sandbox
	// Manifests are the indexer's trunk manifest digests by module, and Verdicts its refusals of them: the catalogue the
	// sandbox composes with, before the case's own manifests are laid over.
	Manifests TrunkManifests
	Verdicts  func() map[string]delivery.ManifestVerdict
}

// composerHead is the node-engine member's head of a case, or empty.
func composerHead(c *Case) string {
	if m := c.member(nodeEngineRepo); m != nil {
		return m.Head
	}
	return ""
}

// Compose asks every placed node and judges their answers (JudgeEngines). An error is only a case it cannot ask at all.
func (e EngineCompositions) Compose(ctx context.Context, c *Case) (delivery.CompositionVerdict, error) {
	began := time.Now()
	key, err := delivery.CompositionKey(c.Group, c.HeadsHash)
	if err != nil {
		return delivery.CompositionVerdict{}, err
	}
	var nodes []string
	if e.Placed != nil {
		if nodes, err = e.Placed(); err != nil {
			return delivery.CompositionVerdict{}, fmt.Errorf("the placement cannot be read, so no node can be asked: %w", err)
		}
	}
	bound := e.Bound
	if bound <= 0 {
		bound = delivery.ComposeAnswerBound
	}
	head := composerHead(c)
	sandboxed := head != "" && e.Sandbox != nil
	answers := map[string]delivery.ComposeCandidateAnswer{}
	silent := map[string]string{}
	var mu sync.Mutex
	var wg sync.WaitGroup
	for _, node := range nodes {
		wg.Add(1)
		go func() {
			defer wg.Done()
			at, cancel := context.WithTimeout(ctx, bound)
			defer cancel()
			a, err := e.Asker.ComposeCandidate(at, node, delivery.ComposeCandidateAsk{Candidate: key, Input: sandboxed})
			if err == nil && sandboxed && a.Verdict != delivery.ComposeCandidateUnready {
				// The composer's build and run are the check's to bound, not the engine's answer bound. A composer that
				// cannot run is no silent node (ADR 0593 §9 lets only that pass): the node's composition fails, naming
				// the head.
				if composed, serr := e.inSandbox(ctx, c, head, node, a); serr != nil {
					a = delivery.ComposeCandidateAnswer{V: a.V, Node: node, Candidate: a.Candidate, Verdict: delivery.ComposeCandidateRefused,
						Refused: "the composer at the node-engine's head " + head + " could not compose it: " + serr.Error()}
				} else {
					a = composed
				}
			}
			mu.Lock()
			defer mu.Unlock()
			switch {
			case err != nil:
				silent[node] = err.Error()
			case a.Verdict == delivery.ComposeCandidateUnready:
				silent[node] = "unready"
			default:
				answers[node] = a
			}
		}()
	}
	wg.Wait()
	// What composed, named by the node-engine's commit: the member's head in the sandbox, else the one the mesh runs.
	composerName := "node-engine"
	if sandboxed {
		composerName = head
	} else if e.Composer != nil && e.Composer() != "" {
		composerName = e.Composer()
	}
	v := JudgeEngines(c.Group, c.HeadsHash, composerName, nodes, answers, silent)
	if head != "" && e.Sandbox == nil {
		v.Warnings = append(v.Warnings, "the group changes the node-engine's composition code and was composed by the running engines")
	}
	v.Took = time.Since(began).Milliseconds()
	return v, nil
}

// JudgeEngines is the check's composition verdict from the engines' answers (ADR 0593 §7, §9): failed when any node's
// answer is refused, naming each node and each module refused with its words (the first refusing node is the machine);
// passed otherwise. Every node in nodes without an answer (silent, by node with why, for the journal only) is named
// with delivery.ComposeNoAnswer as a warning. composer names what composed, for the line. Pure.
func JudgeEngines(group, heads, composerName string, nodes []string, answers map[string]delivery.ComposeCandidateAnswer, silent map[string]string) delivery.CompositionVerdict {
	v := delivery.CompositionVerdict{Group: group, Heads: heads, State: delivery.CompositionPass, SDK: composerName}
	sorted := append([]string(nil), nodes...)
	sort.Strings(sorted)
	var fails []string
	for _, node := range sorted {
		a, answered := answers[node]
		if !answered {
			v.Warnings = append(v.Warnings, node+": "+delivery.ComposeNoAnswer)
			continue
		}
		v.Machines = append(v.Machines, node)
		if a.Verdict != delivery.ComposeCandidateRefused {
			continue
		}
		if v.Machine == "" {
			v.Machine = node
		}
		var words []string
		if a.Refused != "" {
			words = append(words, a.Refused)
		}
		for _, m := range append(append([]delivery.ModuleComposed(nil), a.Modules...), a.Own) {
			if m.State == delivery.ModuleRefusedState {
				words = append(words, m.Module+": "+m.Words)
			}
		}
		if len(words) == 0 {
			words = append(words, "refused, naming no module")
		}
		fails = append(fails, "not composed on "+node+": "+strings.Join(words, "; "))
	}
	if len(fails) > 0 {
		v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
	}
	if len(sorted) == 0 {
		v.Warnings = append(v.Warnings, "no node is placed, so the change was composed on none")
	}
	return v
}

// inSandbox composes a node's answer again with the composer built from the node-engine member's head, over the input the
// node's engine composed from: the node's answer becomes the sandbox's, refused with the composer's words where the
// composer refuses the node.
func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node string, a delivery.ComposeCandidateAnswer) (delivery.ComposeCandidateAnswer, error) {
	if len(a.Input) == 0 {
		return a, fmt.Errorf("the engine of %s answered no composition input, so the node-engine's head cannot compose it", node)
	}
	manifests := map[string]composer.Manifest{}
	if e.Manifests != nil {
		for name, digest := range e.Manifests.ManifestDigests() {
			manifests[name] = composer.Manifest{Digest: digest}
		}
	}
	for _, p := range c.Pulls {
		for name, m := range p.Fps {
			if m.Manifest != "" {
				manifests[name] = composer.Manifest{Digest: m.Manifest}
			}
		}
	}
	verdicts := map[string]delivery.ManifestVerdict{}
	if e.Verdicts != nil {
		for name, v := range e.Verdicts() {
			verdicts[name] = v
		}
	}
	for _, p := range c.Pulls {
		for name := range p.Fps {
			delete(verdicts, name) // a manifest the change brings is the checker's own judge's (ADR 0578 §2)
		}
	}
	req := composer.Request{V: composer.Version, Input: a.Input, Depth: composer.DepthCompose, Machines: []string{node},
		Cases: []composer.Case{{Name: "change", Manifests: manifests, Verdicts: verdicts}}}
	raw, err := json.Marshal(req)
	if err != nil {
		return a, err
	}
	job := delivery.ComposeJob{Fetch: map[string]any{"repository": nodeEngineRepo, "commit": head}, Dir: delivery.RootPath,
		Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
	verdict, err := e.Sandbox.Compose(ctx, job)
	if err != nil {
		return a, fmt.Errorf("the composer at the node-engine's head %s did not answer for %s: %w", head, node, err)
	}
	out := delivery.ComposeCandidateAnswer{V: a.V, Node: node, Candidate: a.Candidate, Verdict: delivery.ComposeCandidateComposed}
	for _, cv := range verdict.Cases {
		if m, ok := cv.Machines[node]; ok && !m.Composes {
			out.Verdict = delivery.ComposeCandidateRefused
			out.Refused = strings.Join(append(append([]string{m.Reason}, m.Problems...), m.Refusals...), "; ")
		}
		for name, read := range cv.Manifests {
			if !read.Accepted {
				out.Verdict = delivery.ComposeCandidateRefused
				out.Modules = append(out.Modules, delivery.ModuleComposed{Module: name, State: delivery.ModuleRefusedState, Words: read.Refused})
			}
		}
	}
	sort.Slice(out.Modules, func(i, j int) bool { return out.Modules[i].Module < out.Modules[j].Module })
	return out, nil
}