mesh-delivery/modules/mesh-delivery-checker/internal/check/engines.go

package check

import (
	"context"
	"encoding/json"
	"fmt"
	"mesh-delivery-checker/rules"
	"sort"
	"strings"
	"sync"
	"time"

	"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
	nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
	ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
	"git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient"
	"git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog"
	"git.novox.be/novox/mesh-delivery/sdk/buildstep"
	"git.novox.be/novox/mesh-delivery/sdk/composer"
)

// **A check's composition is the checker's, made from what each node's engine answers** (novox/hq ADR 0593 §7, §9).
// Only a node knows whether a change composes there: the checker asks every placed node's engine
// `node-engine.compose-candidate` in parallel, with the check's candidate, and waits at most its answer bound
// (compose-answer-bound, 60 s by default). A node that refuses fails the check, naming the node and each module with the
// engine's words; a node that does not answer within the bound, answers unready or cannot be asked is named, "not composed
// on that node: no answer", and the check passes with that warning: its own composition judges again when it applies,
// and the canary rule holds. The checker decides by the answer's verdict, never by its words.
//
// **A group that changes the node-engine's composition code** (a member of the node-engine's repository) is not judged by
// the running engines, which cannot judge their own next code: each node's engine answers the composition input it
// composed from (ne.ComposeCandidateInput.Input), and the composer program built from that member's head composes it in a
// build module's sandbox (Sandbox), with every manifest of the catalogue as the case leaves it and the indexer's verdicts.

// EngineAsker asks one node's engine to compose a check's candidate (the verb node-engine.compose-candidate on that node).
type EngineAsker interface {
	ComposeCandidate(ctx context.Context, node string, ask ne.ComposeCandidateInput) (ne.ComposeCandidateAnswer, error)
}

// Sandbox runs a compose job on the Go build seat and answers the composer's verdict.
type Sandbox interface {
	Compose(ctx context.Context, job buildstep.ComposeJob) (composer.Verdict, error)
}

// EngineCompositions is the checker's composition of a check (Examiner.Compositions).
type EngineCompositions struct {
	Asker EngineAsker
	// Placed names every node in assignments now (the seat mesh-module-assigner's state), or why it cannot be read: then no node
	// can be asked, and the composition is not made (an error), never passed with nobody asked.
	Placed func() ([]string, error)
	// Bound is the answer bound (the setting compose-answer-bound); zero is nec.AnswerBound.
	Bound time.Duration
	// Composer is the node-engine's trunk head: a check whose group has no node-engine member is composed by the composer
	// built there, so every check composes with an acting part built from the change's heads or their trunk, never with
	// the running engines' own code (ADR 0656 §5; the lead's ruling Q3 on milestone 47/G). Nil, or an empty head, leaves
	// such a check composed by the running engines, said.
	Composer func() string
	// Sandbox runs the composer of a node-engine member's head; nil leaves such a group judged by the running engines,
	// said in a warning.
	Sandbox Sandbox
	// Manifests are the indexer's trunk manifest digests by module, and Verdicts its refusals of them: the catalogue the
	// sandbox composes with, before the case's own manifests are laid over.
	Manifests TrunkManifests
	// Published names the latest published version of every mesh package (Examiner.Published): the composer is built
	// against them and the group's heads, as a check is (issue 1921); nil names the group's heads alone.
	Published interface {
		LatestPackages() buildstep.PackageVersions
	}
	Verdicts func() map[string]meshcatalog.VerdictsByDigestValue
	// Requests keeps each request sent to the composer, per check and node (issue 1921); nil keeps none. A request that
	// cannot be kept fails the node's composition, naming why: never dropped.
	Requests interface {
		Keep(ctx context.Context, key, node string, body []byte) error
	}
}

// composerHead is the node-engine member's head of a case, or empty.
// atWord names where a composer was built: the member's head or the node-engine's trunk.
func atWord(member bool) string {
	if member {
		return "head"
	}
	return "trunk"
}

func composerHead(c *Case) string {
	if m := c.member(nodeEngineRepo); m != nil {
		return m.Head
	}
	return ""
}

// Compose asks every placed node and answers their answers, for the rules to judge (rules.JudgeEngines). An error is only a
// case it cannot ask at all.
func (e EngineCompositions) Compose(ctx context.Context, c *Case) (rules.Composed, error) {
	began := time.Now()
	key, err := nodeengineclient.CompositionKey(c.Group, c.Heads)
	if err != nil {
		return rules.Composed{}, err
	}
	var nodes []string
	if e.Placed != nil {
		if nodes, err = e.Placed(); err != nil {
			return rules.Composed{}, fmt.Errorf("the assignments cannot be read, so no node can be asked: %w", err)
		}
	}
	bound := e.Bound
	if bound <= 0 {
		bound = nec.AnswerBound
	}
	// One composition path (ADR 0656 §5): the composer built at the node-engine member's head, else at its trunk.
	head, member := composerHead(c), true
	if head == "" && e.Composer != nil {
		head, member = e.Composer(), false
	}
	sandboxed := head != "" && e.Sandbox != nil
	answers := map[string]ne.ComposeCandidateAnswer{}
	silent := map[string]string{}
	var mu sync.Mutex
	var wg sync.WaitGroup
	for _, node := range nodes {
		wg.Add(1)
		go func() {
			defer wg.Done()
			at, cancel := context.WithTimeout(ctx, bound)
			defer cancel()
			a, err := e.Asker.ComposeCandidate(at, node, ne.ComposeCandidateInput{Candidate: key, Input: sandboxed})
			if err == nil && sandboxed && a.Verdict != nec.Unready {
				// The composer's build and run are the check's to bound, not the engine's answer bound. A composer that
				// cannot run is no silent node (ADR 0593 §9 lets only that pass): the node's composition fails, naming
				// the head.
				if composed, serr := e.inSandbox(ctx, c, head, node, a); serr != nil {
					a = ne.ComposeCandidateAnswer{V: a.V, Node: node, Candidate: a.Candidate, Verdict: nec.Refused,
						Refused: "the composer at the node-engine's " + atWord(member) + " " + head + " could not compose it: " + serr.Error()}
				} else {
					a = composed
				}
			}
			mu.Lock()
			defer mu.Unlock()
			switch {
			case err != nil:
				silent[node] = err.Error()
			case a.Verdict == nec.Unready:
				silent[node] = "unready"
			default:
				answers[node] = a
			}
		}()
	}
	wg.Wait()
	// What composed, named by the node-engine's commit: the member's head in the sandbox, else the one the mesh runs.
	composerName := "node-engine"
	if sandboxed {
		composerName = head
	}
	v := rules.Composed{Group: c.Group, Heads: c.HeadsHash, Composer: composerName, Nodes: nodes, Answers: answers, Silent: silent}
	switch {
	case member && head != "" && e.Sandbox == nil:
		v.Warnings = append(v.Warnings, "the group changes the node-engine's composition code and was composed by the running engines")
	case !sandboxed && e.Sandbox != nil && len(nodes) > 0:
		v.Warnings = append(v.Warnings, "composed by the running engines: no composer at the node-engine's trunk is known to build")
	}
	v.Took = time.Since(began).Milliseconds()
	return v, nil
}

// inSandbox composes a node's answer again with the composer built from the node-engine member's head, over the input the
// node's engine composed from: the node's answer becomes the sandbox's, refused with the composer's words where the
// composer refuses the node.
func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node string, a ne.ComposeCandidateAnswer) (ne.ComposeCandidateAnswer, error) {
	if len(a.Input) == 0 {
		return a, fmt.Errorf("the engine of %s answered no composition input, so the node-engine's head cannot compose it", node)
	}
	manifests := map[string]composer.Manifest{}
	if e.Manifests != nil {
		for name, digest := range e.Manifests.ManifestDigests() {
			manifests[name] = composer.Manifest{Digest: digest}
		}
	}
	for _, p := range c.Pulls {
		for name, m := range p.Fps {
			if m.Manifest != "" {
				manifests[name] = composer.Manifest{Digest: m.Manifest}
			}
		}
	}
	verdicts := map[string]json.RawMessage{}
	if e.Verdicts != nil {
		for name, v := range e.Verdicts() {
			raw, err := json.Marshal(v)
			if err != nil {
				return a, fmt.Errorf("the verdict of the manifest of %s cannot be encoded: %w", name, err)
			}
			verdicts[name] = raw
		}
	}
	// A manifest the change brings is the check's own judge's (ADR 0578 §2, §3): its verdict there when the judge gave one,
	// else none, which the composer reads as accepted; never the running indexer's (F5 of milestone 47/G).
	for _, p := range c.Pulls {
		for name := range p.Fps {
			delete(verdicts, name)
			if raw, ok := c.Judged[name]; ok {
				verdicts[name] = raw
			}
		}
	}
	// The answer carries its input as the object the seat's contract names, decoded with its numbers as written
	// (json.Number), so encoding it again gives the composer the same values the engine sent.
	input, err := json.Marshal(a.Input)
	if err != nil {
		return a, fmt.Errorf("the engine of %s answered a composition input that cannot be encoded again: %w", node, err)
	}
	req := composer.Request{V: composer.Version, Input: input, Depth: composer.DepthCompose, Machines: []string{node},
		Cases: []composer.Case{{Name: "change", Manifests: manifests, Verdicts: verdicts}}, Seats: c.Seats}
	raw, err := json.Marshal(req)
	if err != nil {
		return a, err
	}
	if e.Requests != nil {
		key, err := nodeengineclient.CompositionKey(c.Group, c.Heads)
		if err == nil {
			err = e.Requests.Keep(ctx, key, node, raw)
		}
		if err != nil {
			return a, err
		}
	}
	job := buildstep.ComposeJob{Fetch: buildstep.Fetch{Repository: nodeEngineRepo, Commit: head, Packages: packagesFor(e.Published, c, nodeEngineRepo)}, Dir: meshcatalogclient.RootPath,
		Package: composer.Package, Request: raw, Reader: "the node-engine head of " + c.Group}
	// The ask travels on the bus, whose message limit a whole request passes (1.2 MB for a large node): it goes packed.
	if err := job.PackRequest(raw); err != nil {
		return a, err
	}
	verdict, err := e.Sandbox.Compose(ctx, job)
	if err != nil {
		return a, fmt.Errorf("the composer at the node-engine's head %s did not answer for %s: %w", head, node, err)
	}
	out := ne.ComposeCandidateAnswer{V: a.V, Node: node, Candidate: a.Candidate, Verdict: nec.Composed}
	notJudged := ""
	for _, cv := range verdict.Cases {
		if m, ok := cv.Machines[node]; ok && m.NotJudged != "" {
			notJudged = m.NotJudged // the composer did not judge the node: neither composed nor refused (ADR 0593 §9)
		} else if ok && !m.Composes {
			out.Verdict = nec.Refused
			out.Refused = strings.Join(append(append([]string{m.Reason}, m.Problems...), m.Refusals...), "; ")
		}
		for name, read := range cv.Manifests {
			if !read.Accepted {
				out.Verdict = nec.Refused
				out.Modules = append(out.Modules, ne.Module{Module: name, State: nec.ModuleRefused, Words: read.Refused})
			}
		}
	}
	sort.Slice(out.Modules, func(i, j int) bool { return out.Modules[i].Module < out.Modules[j].Module })
	if notJudged != "" && out.Verdict == nec.Composed {
		out.Verdict, out.Refused = nec.NotJudged, notJudged
	}
	return out, nil
}