mesh-delivery/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/standing_test.go

package main

// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"log"
	"strings"
	"testing"
	"time"

	"git.novox.be/novox/mesh-sdk/go/process"

	"github.com/nats-io/nats.go/jetstream"

	"git.novox.be/novox/mesh-sdk/go/delivery"
	"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)

func gateReport(t *testing.T, node, version string) (string, []byte) {
	t.Helper()
	key, err := delivery.ToolchainKey(node, "typescript")
	if err != nil {
		t.Fatal(err)
	}
	value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
	if err != nil {
		t.Fatal(err)
	}
	return key, value
}

func agreedOf(set *buildrun.ToolchainSet) (string, string) {
	tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
	return tc.Version, none
}

// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
	set := &buildrun.ToolchainSet{}
	var said []string
	gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
	for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
		key, value := gateReport(t, node, version)
		gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
	}
	if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
		t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
	}
	for _, node := range []string{"g14", "novox", "shanks"} {
		gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false, time.Time{})
	}
	gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false, time.Time{})
	gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false, time.Time{}) // another module's: no matter
	if !gate.Settled() {
		t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
	}
	if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
		t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
	}
	if len(said) != 1 {
		t.Fatalf("ace's report is said left out once; said %v", said)
	}
	// ace reports again while unassigned (an old value seen again): still left out.
	key, value := gateReport(t, "ace", "v26.12.0")
	if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
		t.Fatal("an unassigned node's report changed what was passed on")
	}
	// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
	if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false, time.Time{}) {
		t.Fatal("assigning ace again passed nothing on")
	}
	if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
		t.Fatalf("builders that run different versions still disagree; got %q", none)
	}
	// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
	if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false, time.Time{}) {
		t.Fatal("a pending assignment is no assignment on the machine")
	}
	if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
		t.Fatalf("got %q, %q", v, none)
	}
	gate.Assignment("ace.mesh-build-node", nil, true, time.Time{})
	if v, _ := agreedOf(set); v != "v26.12.1" {
		t.Fatalf("got %q", v)
	}
	// A report deleted from the bucket goes whether its node is assigned or not.
	key, _ = gateReport(t, "g14", "v26.12.1")
	if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
		t.Fatal("deleting g14's report passed nothing on")
	}
	if n := len(set.Entries("typescript")); n != 2 {
		t.Fatalf("novox and shanks are left; got %d entries", n)
	}

	// 1945 (a part exits at start for a bucket it works without): a bus that refuses the assignments' watch, or every
	// toolchains bucket, leaves the watches unstarted and the part running, said; a refused bucket it cannot work without
	// is an error naming the bucket and its grant.
	var logs bytes.Buffer
	logger := log.New(&logs, "", 0)
	refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-controller_assignments")}
	if _, err := watchAssignments(context.Background(), refused, newToolchainGate(nil, set.Put, t.Logf), func(context.Context) {}, logger); err != nil {
		t.Errorf("a refused watch of the assignments stopped the start: %v", err)
	}
	if err := newToolchainWatch(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, set, func(context.Context) {}, logger).start(); err != nil {
		t.Errorf("a refused toolchains bucket stopped the start: %v", err)
	}
	if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
		t.Errorf("the refusals were not said: %q", logs.String())
	}
	if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
		!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
		t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
	}
	// The gate's source (the previews' trunk heads): refused at start, the builder goes on, and a candidate asking a trunk
	// head is told why and opens it again then.
	src := &trunkSource{ctx: context.Background(), js: refusingBus{refuseOpen: true, err: refused.err}, heads: &sourceHeads{}, errs: make(chan error, 1), logf: logger.Printf}
	src.start()
	if _, err := src.head(context.Background(), 7, "novox/mesh-catalog"); err == nil || !strings.Contains(err.Error(), delivery.SourceBucket) {
		t.Errorf("a trunk head with the source refused: %v, want why", err)
	}
	if src.opened {
		t.Error("a refused source counts as opened, so it is never opened again")
	}
}

// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
type refusingBus struct {
	refuseOpen bool
	err        error
}

func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
	if r.refuseOpen {
		return nil, r.err
	}
	return refusedWatch{err: r.err}, nil
}

type refusedWatch struct {
	jetstream.KeyValue
	err error
}

func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
	return nil, r.err
}
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
	return nil, r.err
}

// A build node's report counts from the first one it writes since its assignment: a newly assigned node that has not
// reported changes nothing, and the report an earlier run left on the bus (issue 1940, 2026-10-25 20:56: ace.typescript
// v26.12.0 of 2026-10-23 06:29, mesh-build-node assigned to ace again at 20:56:45) is left out, said once, until the node
// writes it again. An assignment whose value names no time stands since the revision that opened it.
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
	left := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC)      // ace's report from its earlier run
	assigned := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node assigned to ace again
	report := func(t *testing.T, node, version string, at time.Time) (string, []byte) {
		key, _ := gateReport(t, node, version)
		value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
		if err != nil {
			t.Fatal(err)
		}
		return key, value
	}
	for _, c := range []struct {
		name       string
		assignment string    // ace's assignment value
		revision   time.Time // when the bus wrote it
		at         time.Time // when ace wrote its report
		counts     bool
	}{
		{"no report yet, then one", `{}`, time.Time{}, time.Now(), true},
		{"a report left before the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, left, false},
		{"a report left before a value with no time", `{"node":"ace","module":"mesh-build-node"}`, assigned, left, false},
		{"a report written since the assignment", `{"node":"ace","module":"mesh-build-node","assigned":"2026-10-25T19:56:45Z"}`, assigned, assigned.Add(30 * time.Second), true},
	} {
		t.Run(c.name, func(t *testing.T) {
			set := &buildrun.ToolchainSet{}
			var said []string
			gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) })
			for _, node := range []string{"g14", "novox", "shanks"} {
				gate.Assignment(node+".mesh-build-node", []byte(`{}`), false, time.Time{})
				key, value := gateReport(t, node, "v26.12.1")
				gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
			}
			gate.Settled()
			if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision) {
				t.Fatal("an assignment with no report passed something on")
			}
			key, value := report(t, "ace", "v26.12.0", c.at)
			if got := gate.Toolchain(buildrun.NodeBuildModule, key, value, false); got != c.counts {
				t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
			}
			// A rewrite of the assignment that does not move it changes nothing, and nothing is said again.
			if gate.Assignment("ace.mesh-build-node", []byte(c.assignment), false, c.revision.Add(time.Minute)) {
				t.Fatal("the same assignment written again changed what was passed on")
			}
			v, none := agreedOf(set)
			if c.counts {
				if !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
					t.Fatalf("ace's own report counts and disagrees; got %q, %q", v, none)
				}
				return
			}
			if v != "v26.12.1" || none != "" {
				t.Fatalf("ace's left report is no builder's, so g14, novox and shanks agree; got %q, %q", v, none)
			}
			if len(said) != 1 || !strings.Contains(said[0], "before mesh-build-node's assignment on ace at 2026-10-25T19:56:45Z") {
				t.Fatalf("said %q, want once why ace's report is left out", said)
			}
			// ace writes its report again since it was assigned: it counts.
			key, value = report(t, "ace", "v26.12.1", assigned.Add(time.Minute))
			if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
				t.Fatal("ace's new report was not passed on")
			}
			if n := len(set.Entries("typescript")); n != 4 {
				t.Fatalf("four builders report; got %d", n)
			}
		})
	}
}