mesh-delivery/modules/mesh-delivery-checker/internal/check/engines_test.go

package check

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"mesh-delivery-checker/rules"
	"strings"
	"sync"
	"testing"
	"time"

	"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
	nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
	ne "git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliverycheckerclient"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer"
	"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk"
)

// fakeEngines answers compose-candidate by node from memory; a node it holds no answer for never answers.
type fakeEngines struct {
	mu      sync.Mutex
	answers map[string]ne.ComposeCandidateAnswer
	errs    map[string]error
	asked   map[string]ne.ComposeCandidateInput
}

func (f *fakeEngines) ComposeCandidate(ctx context.Context, node string, ask ne.ComposeCandidateInput) (ne.ComposeCandidateAnswer, error) {
	f.mu.Lock()
	if f.asked == nil {
		f.asked = map[string]ne.ComposeCandidateInput{}
	}
	f.asked[node] = ask
	a, ok := f.answers[node]
	err := f.errs[node]
	f.mu.Unlock()
	if err != nil {
		return ne.ComposeCandidateAnswer{}, err
	}
	if !ok {
		<-ctx.Done()
		return ne.ComposeCandidateAnswer{}, ctx.Err()
	}
	return a, nil
}

// fakeSandbox answers a compose job with a verdict, recording the job.
type fakeSandbox struct {
	mu   sync.Mutex
	jobs []buildstep.ComposeJob
	v    func(node string) composer.Verdict
	err  error
}

func (f *fakeSandbox) Compose(_ context.Context, job buildstep.ComposeJob) (composer.Verdict, error) {
	var req composer.Request
	body, err := job.RequestBytes()
	if err != nil {
		return composer.Verdict{}, err
	}
	if err := json.Unmarshal(body, &req); err != nil {
		return composer.Verdict{}, err
	}
	f.mu.Lock()
	f.jobs = append(f.jobs, job)
	f.mu.Unlock()
	if f.err != nil {
		return composer.Verdict{}, f.err
	}
	return f.v(req.Machines[0]), nil
}

// keptRequests is the state of composition requests in memory; a put fails with err when it is set.
type keptRequests struct {
	mu   sync.Mutex
	vals map[string][]byte
	err  error
}

func (k *keptRequests) Put(_ context.Context, key string, value []byte) (uint64, error) {
	k.mu.Lock()
	defer k.mu.Unlock()
	if k.err != nil {
		return 0, k.err
	}
	if k.vals == nil {
		k.vals = map[string][]byte{}
	}
	k.vals[key] = value
	return 1, nil
}

func (k *keptRequests) read() StateRead {
	return StateRead{Keys: func() ([]string, error) {
		var out []string
		for n := range k.vals {
			out = append(out, n)
		}
		return out, nil
	}, Get: func(key string) (json.RawMessage, error) { return k.vals[key], nil }}
}

type digests map[string]string

func (d digests) ManifestDigests() map[string]string { return d }

// ADR 0593 §7, §9 (D12, G1, G2): the checker asks every placed node's engine in parallel and makes the composition from
// their answers: a refusal fails it naming the node and each module; a node that does not answer within the bound,
// answers unready or cannot be asked is named "not composed on that node: no answer" and the check passes with that
// warning, decided by the verdict and never by its words; a group with a node-engine member is composed by the composer
// built at that member's head, over the input each node's engine composed from.
func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) {
	composed := func(node string) ne.ComposeCandidateAnswer {
		return ne.ComposeCandidateAnswer{Node: node, Verdict: nec.Composed,
			Modules: []ne.Module{{Module: "web", State: nec.ModuleComposed, Digest: "sha256:" + strings.Repeat("a", 64)}},
			Own:     ne.Module{Module: nec.Own, State: nec.ModuleComposed, Digest: "sha256:" + strings.Repeat("b", 64)}}
	}
	cs := caseOf(t, pullOf(repoA, 7, head1))
	key, _ := nodeengineclient.CompositionKey(cs.Group, cs.HeadsHash)
	var last rules.Composed // the engines' answers of the last compose, for the check's verdict to take in
	compose := func(e *fakeEngines, nodes ...string) meshwants.CompositionsByGroupHeadsValue {
		t.Helper()
		ec := EngineCompositions{Asker: e, Placed: func() ([]string, error) { return nodes, nil }, Bound: 50 * time.Millisecond,
			Composer: func() string { return strings.Repeat("7", 40) }}
		raw, err := ec.Compose(context.Background(), cs)
		if err != nil {
			t.Fatal(err)
		}
		last = raw
		return rules.JudgeEngines(raw)
	}

	t.Run("one node refusing fails the composition, naming the node and the module", func(t *testing.T) {
		refusing := composed("laptop")
		refusing.Verdict = nec.Refused
		refusing.Modules = []ne.Module{{Module: "web", State: nec.ModuleRefused, Words: "a required value is unset"},
			{Module: "mail", State: nec.ModuleHeld, Words: "held as today"}}
		e := &fakeEngines{answers: map[string]ne.ComposeCandidateAnswer{"anchor": composed("anchor"), "laptop": refusing}}
		v := compose(e, "anchor", "laptop")
		if v.State != meshdeliverycheckerclient.CompositionFail || v.Machine != "laptop" || !strings.Contains(v.Words, "not composed on laptop: web: a required value is unset") ||
			strings.Contains(v.Words, "mail") || len(v.Machines) != 2 {
			t.Errorf("verdict = %+v", v)
		}
		if e.asked["anchor"].Candidate != key || e.asked["laptop"].Input {
			t.Errorf("asked %+v, want the check's candidate %s and no input", e.asked, key)
		}
		var ver meshdeliverychecker.VerdictsByGroupHeadsValue
		rules.JudgeComposition(&ver, rules.Composition{Composed: &last})
		if ver.State != meshdeliverycheckerclient.VerdictFail || ver.Failed.Machine != "laptop" {
			t.Errorf("the check's verdict = %+v", ver)
		}
	})

	t.Run("a silent, unready or unreachable node is no answer, and the check passes naming it", func(t *testing.T) {
		unready := ne.ComposeCandidateAnswer{Node: "g14", Verdict: nec.Unready, Refused: "a module refused it"}
		e := &fakeEngines{answers: map[string]ne.ComposeCandidateAnswer{"anchor": composed("anchor"), "g14": unready},
			errs: map[string]error{"ace": errors.New("no responders")}}
		began := time.Now()
		v := compose(e, "anchor", "laptop", "g14", "ace")
		if took := time.Since(began); took > 2*time.Second {
			t.Errorf("the nodes were not asked in parallel within the bound: %s", took)
		}
		// No sandbox: the running engines composed, named so, though the node-engine's trunk head is known.
		if v.State != meshdeliverycheckerclient.CompositionPass || v.Sdk != "node-engine" || len(v.Machines) != 1 {
			t.Fatalf("verdict = %+v", v)
		}
		for _, node := range []string{"ace", "g14", "laptop"} {
			if !strings.Contains(strings.Join(v.Warnings, "; "), node+": "+nec.NoAnswer) {
				t.Errorf("%s is not named: %v", node, v.Warnings)
			}
		}
		var ver meshdeliverychecker.VerdictsByGroupHeadsValue
		ver.State = meshdeliverycheckerclient.VerdictPass
		rules.JudgeComposition(&ver, rules.Composition{Composed: &last})
		if ver.State != meshdeliverycheckerclient.VerdictPass || !strings.Contains(composition(ver.Composition, 0, false), "laptop: "+nec.NoAnswer) {
			t.Errorf("the line = %q", composition(ver.Composition, 0, false))
		}
	})

	t.Run("a node-engine member is composed by the composer at its head over each node's input", func(t *testing.T) {
		c, err := meshcatalog.DecodeCandidatesByGroupValue(candidateOf(t, "feat/x", pullOf(nodeEngineRepo, 3, head2), pullOf(repoA, 7, head1)))
		if err != nil {
			t.Fatal(err)
		}
		host := CaseOf(c, map[string]string{repoA: trunkA, nodeEngineRepo: trunkA})
		host.Seats = []json.RawMessage{json.RawMessage(`{"seat":"node-x","scope":"node"}`)} // the heads' list (ADR 0636 §4)
		withInput := composed("anchor")
		withInput.Input = map[string]any{"nodes": []any{map[string]any{"name": "anchor"}}}
		// Issue 1966 (the group try of 2026-11-06 10:07): the composer at the head composes against the check's seat list.
		host.Seats = []json.RawMessage{json.RawMessage(`{"format":1,"seat":"node-uplink"}`)}
		e := &fakeEngines{answers: map[string]ne.ComposeCandidateAnswer{"anchor": withInput}}
		sb := &fakeSandbox{v: func(node string) composer.Verdict {
			return composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: "change",
				Machines: map[string]composer.Machine{node: {Composes: false, Reason: "the new composer refuses it"}}}}}
		}}
		ec := EngineCompositions{Asker: e, Placed: func() ([]string, error) { return []string{"anchor"}, nil }, Bound: time.Second,
			Sandbox: sb, Manifests: digests{"web": "sha256:" + strings.Repeat("c", 64)}, Published: published{}}
		raw, err := ec.Compose(context.Background(), host)
		v := rules.JudgeEngines(raw)
		if err != nil {
			t.Fatal(err)
		}
		if !e.asked["anchor"].Input || len(sb.jobs) != 1 {
			t.Fatalf("asked %+v, jobs %+v: want the input asked and one job", e.asked, sb.jobs)
		}
		job := sb.jobs[0]
		var req composer.Request
		if len(job.Request) == 0 || job.Request[0] != '"' {
			t.Errorf("the request went whole, not packed (issue 1966): %.60s", job.Request)
		}
		body, _ := job.RequestBytes()
		_ = json.Unmarshal(body, &req)
		if job.Fetch.Repository != nodeEngineRepo || job.Fetch.Commit != head2 || job.Package != composer.Package ||
			string(req.Input) != `{"nodes":[{"name":"anchor"}]}` || req.Cases[0].Manifests["web"].Digest == "" ||
			len(req.Seats) != 1 || string(req.Seats[0]) != string(host.Seats[0]) {
			t.Errorf("job = %+v, request %+v", job, req)
		}
		// Issue 1921 (notes 66-67): the request sent to the composer is kept per check and node, exactly as sent (read back
		// whole through the tool's reader); a request that cannot be kept fails the node's composition, naming why.
		store := &keptRequests{}
		ec.Requests = RequestKeeper{Store: store}
		if _, err := ec.Compose(context.Background(), host); err != nil {
			t.Fatal(err)
		}
		keyOfHost, _ := nodeengineclient.CompositionKey(host.Group, host.Heads)
		lastBody, _ := sb.jobs[len(sb.jobs)-1].RequestBytes()
		got, err := ReadRequests(store.read(), keyOfHost, "anchor")
		if b, _ := json.Marshal(got); err != nil || !strings.Contains(string(b), `"request":`+string(lastBody)) {
			t.Errorf("kept %s, %v: want the request of the last job under %s/anchor", b, err, keyOfHost)
		}
		store.err = errors.New("the state is full")
		raw, _ = ec.Compose(context.Background(), host)
		if v := rules.JudgeEngines(raw); v.State != meshdeliverycheckerclient.CompositionFail || !strings.Contains(v.Words, "could not be kept") || !strings.Contains(v.Words, "the state is full") || !strings.Contains(v.Words, RequestsDropTool) {
			t.Errorf("a request that cannot be kept gave %+v, which must name %s", v, RequestsDropTool)
		}
		// The drop tool: records the hand act first (a drop whose act is not recorded is not made), then drops that check's
		// requests and the pieces no other kept request names; another check's stay.
		store.err = nil
		other := RequestKeeper{Store: store}
		if err := other.Keep(context.Background(), "g.other", "anchor", []byte("another check's request")); err != nil {
			t.Fatal(err)
		}
		drop := StateDrop{StateRead: store.read(), Delete: func(k string) error { store.mu.Lock(); delete(store.vals, k); store.mu.Unlock(); return nil }}
		before := len(store.vals)
		if _, err := DropRequests(drop, keyOfHost, "the state is full", func(string) error { return errors.New("no holder") }); err == nil || len(store.vals) != before {
			t.Errorf("a drop whose hand act was not recorded was made: %v, %d of %d kept", err, len(store.vals), before)
		}
		var acts []string
		answer, err := DropRequests(drop, keyOfHost, "the state is full", func(what string) error { acts = append(acts, what); return nil })
		if b, _ := json.Marshal(answer); err != nil || len(acts) != 1 || !strings.Contains(string(b), `"dropped_requests":["anchor"]`) {
			t.Errorf("the drop answered %s, %v, acts %v", b, err, acts)
		}
		if _, still := store.vals[RequestKey(keyOfHost, "anchor")]; still {
			t.Errorf("the dropped check's request is kept still")
		}
		if got, err := ReadRequests(store.read(), "g.other", "anchor"); err != nil || !strings.Contains(fmt.Sprint(got), "another check's request") {
			t.Errorf("another check's request after the drop: %v, %v", got, err)
		}
		if _, err := DropRequests(drop, keyOfHost, "again", func(string) error { return nil }); err == nil {
			t.Errorf("a check with no kept request was dropped")
		}
		ec.Requests = nil
		// Issue 1921: the composer is built at the group's heads, also a member that publishes no package yet.
		if pkgs := job.Fetch.Packages; len(pkgs) != 1 || pkgs[trunk.ForgeModulePrefix+"mesh-delivery"] != head1 {
			t.Errorf("the composer is fetched with the packages %v, want the group's mesh-delivery member at its head %s alone", pkgs, head1)
		}
		if v.State != meshdeliverycheckerclient.CompositionFail || v.Sdk != head2 || !strings.Contains(v.Words, "the new composer refuses it") {
			t.Errorf("verdict = %+v", v)
		}
		// ADR 0593 §9 (ADR 0601 §4's kind): a node the composer did not judge is named as no answer with its words, by the
		// contract's field, and the check passes; words in Reason alone are no such mark.
		sb.err = nil
		for _, row := range []struct {
			m    composer.Machine
			pass bool
		}{
			{composer.Machine{NotJudged: "its node-engine sends no running state for web"}, true},
			{composer.Machine{Reason: "not judged: its node-engine sends no running state for web"}, false},
		} {
			sb.v = func(node string) composer.Verdict {
				return composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: "change", Machines: map[string]composer.Machine{node: row.m}}}}
			}
			raw, err = ec.Compose(context.Background(), host)
			v = rules.JudgeEngines(raw)
			named := len(v.Warnings) == 1 && strings.Contains(v.Warnings[0], "anchor: "+nec.NoAnswer+" (its node-engine sends no running state for web)")
			if err != nil || (v.State == meshdeliverycheckerclient.CompositionPass) != row.pass || named != row.pass {
				t.Errorf("%+v: verdict %+v, %v", row.m, v, err)
			}
		}
		// Review finding 3 (ADR 0593 §9 lets only a node that does not answer pass): a composer at the head that cannot
		// run fails the composition, naming the head, never passing as a silent node.
		sb.err = errors.New("the build of mesh-compose failed")
		raw, err = ec.Compose(context.Background(), host)
		v = rules.JudgeEngines(raw)
		if err != nil || v.State != meshdeliverycheckerclient.CompositionFail || !strings.Contains(v.Words, head2) ||
			!strings.Contains(v.Words, "the build of mesh-compose failed") || len(v.Warnings) != 0 {
			t.Errorf("a composer that cannot run: %+v, %v", v, err)
		}
	})

	// ADR 0578 §2, §3, ADR 0656 §5 (milestone 47/G D12, D13): with no node-engine member, every node is composed by the
	// composer built at the node-engine's trunk over the input its engine answers, and a manifest the change brings carries
	// the check's own judge's verdict, not the running indexer's: one the running indexer refused and the judge accepted
	// composes, and the verdict names the trunk's composer.
	t.Run("a change's manifest the check's judge accepted is composed by the composer at the node-engine's trunk", func(t *testing.T) {
		p := pullOf(repoA, 7, head1)
		p.Modules = []string{"mesh-build-go"}
		p.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"mesh-build-go": {Fp2: "src2:" + strings.Repeat("f", 64), Manifest: strings.Repeat("d", 64)}}
		brings := caseOf(t, p)
		brings.Judged = map[string]json.RawMessage{"mesh-build-go": json.RawMessage(`{"module":"mesh-build-go","state":"accepted"}`)}
		trunkHead := strings.Repeat("7", 40)
		withInput := composed("anchor")
		withInput.Input = map[string]any{"nodes": []any{map[string]any{"name": "anchor"}}}
		e := &fakeEngines{answers: map[string]ne.ComposeCandidateAnswer{"anchor": withInput}}
		sb := &fakeSandbox{v: func(node string) composer.Verdict {
			return composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: "change",
				Machines: map[string]composer.Machine{node: {Composes: true}}}}}
		}}
		indexer := map[string]meshcatalog.VerdictsByDigestValue{"mesh-build-go": {Module: "mesh-build-go", State: "refused", Problems: []string{"serves no verb input"}}, "mail": {Module: "mail", State: "accepted"}}
		ec := EngineCompositions{Asker: e, Placed: func() ([]string, error) { return []string{"anchor"}, nil }, Bound: time.Second,
			Sandbox: sb, Composer: func() string { return trunkHead }, Published: published{},
			Verdicts: func() map[string]meshcatalog.VerdictsByDigestValue { return indexer }}
		raw, err := ec.Compose(context.Background(), brings)
		if err != nil {
			t.Fatal(err)
		}
		if v := rules.JudgeEngines(raw); v.State != meshdeliverycheckerclient.CompositionPass || v.Sdk != trunkHead {
			t.Fatalf("verdict = %+v, want a pass composed at the node-engine's trunk %s", v, trunkHead)
		}
		if len(sb.jobs) != 1 || sb.jobs[0].Fetch.Commit != trunkHead || sb.jobs[0].Fetch.Repository != nodeEngineRepo {
			t.Fatalf("jobs %+v: want one job at the node-engine's trunk", sb.jobs)
		}
		var req composer.Request
		b0, _ := sb.jobs[0].RequestBytes()
		_ = json.Unmarshal(b0, &req)
		if got := string(req.Cases[0].Verdicts["mesh-build-go"]); got != string(brings.Judged["mesh-build-go"]) || len(req.Cases[0].Verdicts) != 2 {
			t.Errorf("the request's verdicts %s: want the check's judge's for mesh-build-go and the indexer's for mail", req.Cases[0].Verdicts)
		}
		// A manifest the judge gave no verdict on carries none, which the composer reads as accepted (ADR 0578 §3).
		brings.Judged = nil
		if _, err := ec.Compose(context.Background(), brings); err != nil {
			t.Fatal(err)
		}
		var none composer.Request
		bn, _ := sb.jobs[len(sb.jobs)-1].RequestBytes()
		_ = json.Unmarshal(bn, &none)
		if _, kept := none.Cases[0].Verdicts["mesh-build-go"]; kept {
			t.Errorf("the running indexer's verdict on the change's manifest was kept: %s", none.Cases[0].Verdicts)
		}
		// A refusal the trunk's composer makes stays the node-engine's own.
		sb.v = func(node string) composer.Verdict {
			return composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: "change",
				Machines: map[string]composer.Machine{node: {Composes: false, Reason: "a required value is unset"}}}}}
		}
		raw, _ = ec.Compose(context.Background(), brings)
		if v := rules.JudgeEngines(raw); v.State != meshdeliverycheckerclient.CompositionFail || !strings.Contains(v.Words, "a required value is unset") {
			t.Errorf("a refusal the trunk's composer makes: %+v", v)
		}
	})

	// Review finding 4: assignments that cannot be read is no mesh with no node: the composition is not made, said.
	t.Run("an unreadable assignments makes no composition", func(t *testing.T) {
		ec := EngineCompositions{Asker: &fakeEngines{}, Placed: func() ([]string, error) { return nil, errors.New("no such bucket") }}
		if _, err := ec.Compose(context.Background(), cs); err == nil || !strings.Contains(err.Error(), "no such bucket") {
			t.Errorf("an unreadable assignments: %v", err)
		}
		b := newBench(t)
		b.ex.Compositions = ec
		v := b.judge(t, cs).Verdict
		if v.Composition == nil || v.Composition.State != meshdeliverycheckerclient.CompositionNotRun || !strings.Contains(v.Composition.Words, "no such bucket") {
			t.Errorf("the verdict's composition = %+v", v.Composition)
		}
	})
}