Build a key only with the toolchain its fingerprint names, accept '.' as a root module's path, and choose only online nodes (issues 1615, 1607 review)

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go

@@ -193,6 +193,13 @@ func startBed(t *testing.T) *bed {
 	b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
 	mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
 	b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
+	presence := mk(delivery.PresenceBucket, delivery.PresenceHistory)
+	for _, node := range []string{"n1", "n2"} {
+		data, _ := delivery.EncodePresence(delivery.Presence{Online: true})
+		if _, err := presence.Put(ctx, node, data); err != nil {
+			t.Fatal(err)
+		}
+	}
 	b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
 	b.fake.serve(t, ctl, "mesh-build-go")
 	b.fake.serve(t, ctl, "mesh-build-node")
@@ -221,6 +228,15 @@ func (b *bed) report(t *testing.T, module, node, language, version, recipe strin
 
 // module writes modules.<name> as the indexer does for a module with a new-form fingerprint, and its manifest by digest.
 func (b *bed) module(t *testing.T, name, fixture, path string) string {
+	return b.moduleWith(t, name, fixture, path, map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}})
+}
+
+// moduleWith is module with the toolchains its fingerprint was computed with.
+func (b *bed) moduleWith(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed) string {
+	return b.moduleCommit(t, name, fixture, path, toolchains, commit)
+}
+
+func (b *bed) moduleCommit(t *testing.T, name, fixture, path string, toolchains map[string]delivery.ToolchainUsed, at string) string {
 	t.Helper()
 	raw, err := os.ReadFile(filepath.Join("testdata", fixture))
 	if err != nil {
@@ -238,8 +254,8 @@ func (b *bed) module(t *testing.T, name, fixture, path string) string {
 	if _, err := manifests.Put(b.ctx, digest, mv); err != nil {
 		t.Fatal(err)
 	}
-	data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: commit, FP: "src1:" + strings.Repeat("f", 64), FP2: fp,
-		Manifest: digest, Path: path})
+	data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: at, FP: "src1:" + strings.Repeat("f", 64), FP2: fp,
+		Manifest: digest, Path: path, Toolchains: toolchains})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -334,10 +350,10 @@ func TestASecondBuilderInstanceDoesNotBuildTheSameSourceAgain(t *testing.T) { //
 	}
 }
 
-// TestAFailedKeyIsBuiltAgainAsANewRevisionWhenAToolchainChanges: a build step fails on one node, so the key is failed
-// naming it; a toolchain report of a node then wakes the builder, which builds the key again as a new revision of the same
-// key, now done.
-func TestAFailedKeyIsBuiltAgainAsANewRevisionWhenAToolchainChanges(t *testing.T) { // 1607/AC6, AC7
+// TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails on one node, so the
+// key is failed naming it; the indexer writing modules.<m> again with the same fingerprint wakes the builder, which builds
+// the key again as a new revision of the same key, now done.
+func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7
 	b := startBed(t)
 	b.fake.failBuild.Store(true)
 	b.module(t, "ca-trust", "catalogue-a67f0196-ca-trust.module.json", "modules/ca-trust")
@@ -352,9 +368,8 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionWhenAToolchainChanges(t *testing.T)
 		t.Fatal(err)
 	}
 	b.fake.failBuild.Store(false)
-	// The build module's recipe moved on both nodes: the toolchain's entries changed, twice (between the two the nodes disagree).
-	b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "2")
-	b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "2")
+	// The indexer writes the module again (here at a later commit), with the same fingerprint.
+	b.moduleAt(t, "ca-trust", "catalogue-a67f0196-ca-trust.module.json", "modules/ca-trust", "go1.27.1", "89abcdef0123456789abcdef0123456789abcdef")
 	b.await(t, "ca-trust", isState(delivery.BuildDone))
 	done, err := b.builds.Get(b.ctx, key)
 	if err != nil {
@@ -364,3 +379,45 @@ func TestAFailedKeyIsBuiltAgainAsANewRevisionWhenAToolchainChanges(t *testing.T)
 		t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision())
 	}
 }
+
+// TestAToolchainChangeDoesNotBuildTheOldFingerprint: the fingerprint names go1.27.1, the nodes then agree on go1.27.2 (a
+// report of both nodes), and the builder writes no key for the old fingerprint although it was woken by the report; a
+// module value naming go1.27.2 (the new fingerprint, computed by the indexer) is built.
+func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/AC4
+	b := startBed(t)
+	b.report(t, "mesh-build-go", "n1", "go", "go1.27.2", "1")
+	b.report(t, "mesh-build-go", "n2", "go", "go1.27.2", "1")
+	b.module(t, "ca-trust", "catalogue-a67f0196-ca-trust.module.json", "modules/ca-trust") // names go1.27.1
+	b.start(t, "n1", nil)
+	// Another module, whose fingerprint names the new toolchain, is built; by then the old one has been looked at.
+	b.moduleAt(t, "ca-trust2", "catalogue-a67f0196-ca-trust.module.json", "modules/ca-trust", "go1.27.2", commit)
+	b.await(t, "ca-trust2", isState(delivery.BuildDone))
+	if _, err := b.builds.Get(b.ctx, mustKey(t, "ca-trust")); err == nil {
+		t.Error("the old fingerprint, naming go1.27.1, was built with go1.27.2")
+	}
+	if n := b.fake.count("mesh-build-go.build_go_build"); n != 2 {
+		t.Errorf("build_go_build was called %d times, want 2 (the new fingerprint on two nodes only)", n)
+	}
+}
+
+// TestARootModuleIsBuilt: a module whose path is "." is given to the build tools as it is.
+func TestARootModuleIsBuilt(t *testing.T) { // 1615/AC4
+	b := startBed(t)
+	b.module(t, "ca-trust", "catalogue-a67f0196-ca-trust.module.json", delivery.RootPath)
+	b.start(t, "n1", nil)
+	b.await(t, "ca-trust", isState(delivery.BuildDone))
+}
+
+func mustKey(t *testing.T, module string) string {
+	k, err := delivery.BuilderBuildKey(module, fp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return k
+}
+
+// moduleAt is module with the go toolchain version the fingerprint names and the commit it was computed at.
+func (b *bed) moduleAt(t *testing.T, name, fixture, path, goVersion, at string) {
+	t.Helper()
+	b.moduleCommit(t, name, fixture, path, map[string]delivery.ToolchainUsed{"go": {Version: goVersion, Recipe: "1"}}, at)
+}

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go

@@ -159,10 +159,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		}
 	}
 	tools := &build.ToolchainSet{}
+	presence := &build.PresenceSet{}
 	b := &build.Builder{
 		Claimer:    build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
 		Runner:     build.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: bound},
 		Toolchains: tools,
+		Presence:   presence,
 		Manifests:  manifestReader{manifests},
 		Nodes:      nodesPerFingerprint,
 		Logf:       logger.Printf,
@@ -189,6 +191,27 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		}
 		defer cc.Stop()
 	}
+	// The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing
+	// builds; the builder says so and is started again with the watchman.
+	presenceErrs := make(chan error, 1)
+	switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
+	case errors.Is(err, jetstream.ErrBucketNotFound):
+		logger.Printf("the bucket %s does not exist yet: no node counts as online until the builder starts again", delivery.PresenceBucket)
+	case err != nil:
+		return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
+	default:
+		pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
+		if err != nil {
+			return err
+		}
+		go func() {
+			presenceErrs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
+				if presence.Put(e.Key(), e.Value(), !opOf(e)) {
+					b.Reconsider(ctx) // a node came online or went: a build that waited for one may go on
+				}
+			})
+		}()
+	}
 	each := func(e jetstream.KeyValueEntry) {
 		if opOf(e) {
 			b.Module(ctx, e.Key(), e.Value())
@@ -211,5 +234,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
 		return err
 	case err := <-watch.errs:
 		return err
+	case err := <-presenceErrs:
+		return err
 	}
 }

modules/mesh-delivery-builder/internal/build/choose.go

@@ -3,28 +3,40 @@ package build
 import (
 	"fmt"
 	"sort"
+	"strings"
 
 	"git.novox.be/novox/mesh-sdk/go/delivery"
 )
 
 // ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the
 // version the nodes agree on, the ones with the fewest steps the builder has running there first, a tie broken by the
-// node's name. It answers the toolchain they agree on. When nothing is reported, or the nodes report different versions
-// (so the indexer agreed none), it fails naming the language and why. The build module's health is not read here: a node
-// whose module refuses on its toolchain refuses the step, and the build fails naming it.
-func ChooseNodes(language string, entries []delivery.NodeToolchain, running map[string]int, want int) ([]string, delivery.Toolchain2, error) {
+// node's name. A node qualifies only when online is true of it (the watchman's presence of the node; nil means every
+// node is). It answers the toolchain all the reporting nodes agree on, online or not, as the indexer computes it. When
+// nothing is reported, or the nodes report different versions (so the indexer agreed none), it fails naming the language
+// and why; when no reporting node is online it fails naming the nodes. A node whose build module refuses on its toolchain
+// refuses the step, and the build fails naming it.
+func ChooseNodes(language string, entries []delivery.NodeToolchain, running map[string]int, online func(node string) bool, want int) ([]string, delivery.Toolchain2, error) {
 	tc, none := delivery.AgreedToolchain(entries)
 	if none != "" {
 		return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none)
 	}
 	nodes := make([]string, 0, len(entries))
+	var reporting []string
 	seen := map[string]bool{}
 	for _, e := range entries {
-		if !seen[e.Node] {
-			seen[e.Node] = true
+		if seen[e.Node] {
+			continue
+		}
+		seen[e.Node] = true
+		reporting = append(reporting, e.Node)
+		if online == nil || online(e.Node) {
 			nodes = append(nodes, e.Node)
 		}
 	}
+	if len(nodes) == 0 {
+		sort.Strings(reporting)
+		return nil, tc, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online", language, strings.Join(reporting, ", "))
+	}
 	sort.Slice(nodes, func(i, j int) bool {
 		if running[nodes[i]] != running[nodes[j]] {
 			return running[nodes[i]] < running[nodes[j]]

modules/mesh-delivery-builder/internal/build/choose_test.go

@@ -17,7 +17,7 @@ func tcs(language, version string, nodes ...string) []delivery.NodeToolchain {
 
 func TestChooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3
 	entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks")
-	nodes, tc, err := ChooseNodes("go", entries, map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}, 2)
+	nodes, tc, err := ChooseNodes("go", entries, map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}, nil, 2)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -30,19 +30,19 @@ func TestChooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T)
 }
 
 func TestChooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4
-	nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, 2)
+	nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2)
 	if err != nil || len(nodes) != 1 || nodes[0] != "novox" {
 		t.Errorf("nodes = %v, err %v, want [novox]", nodes, err)
 	}
 }
 
 func TestChooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4
-	_, _, err := ChooseNodes("typescript", nil, nil, 2)
+	_, _, err := ChooseNodes("typescript", nil, nil, nil, 2)
 	if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) {
 		t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport)
 	}
 	entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...)
-	_, _, err = ChooseNodes("go", entries, nil, 2)
+	_, _, err = ChooseNodes("go", entries, nil, nil, 2)
 	if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") {
 		t.Errorf("disagreeing nodes: error = %v, want both versions named", err)
 	}
@@ -61,3 +61,19 @@ func TestAgreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { /
 		t.Errorf("one node: %q, %v", d, err)
 	}
 }
+
+func TestChooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3
+	entries := tcs("go", "go1.27.1", "ace", "g14", "novox")
+	online := func(n string) bool { return n != "g14" } // a sleeping laptop
+	nodes, tc, err := ChooseNodes("go", entries, map[string]int{"novox": 5}, online, 2)
+	if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" {
+		t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err)
+	}
+	_, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2)
+	if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") {
+		t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err)
+	}
+	if tc.Version != "go1.27.1" {
+		t.Errorf("none online: the agreed toolchain %+v is still answered", tc)
+	}
+}

modules/mesh-delivery-builder/internal/build/presence.go

@@ -0,0 +1,53 @@
+package build
+
+import (
+	"sync"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key `<node>`, the
+// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the
+// watchman has no key of is not online. Safe for use from the watch and from a build at once.
+type PresenceSet struct {
+	mu     sync.Mutex
+	online map[string]bool
+}
+
+// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a
+// value that is not a presence are left out.
+func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool {
+	node, part, _, err := delivery.ParsePresenceKey(key)
+	if err != nil || part != "" {
+		return false
+	}
+	on := false
+	if !deleted {
+		p, err := delivery.DecodePresence(value)
+		if err != nil {
+			return false
+		}
+		on = p.Online
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.online[node] == on {
+		return false
+	}
+	if s.online == nil {
+		s.online = map[string]bool{}
+	}
+	if on {
+		s.online[node] = true
+	} else {
+		delete(s.online, node)
+	}
+	return true
+}
+
+// Online is whether the watchman last saw the node online.
+func (s *PresenceSet) Online(node string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.online[node]
+}

modules/mesh-delivery-builder/internal/build/presence_test.go

@@ -0,0 +1,43 @@
+package build
+
+import (
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+func presenceValue(t *testing.T, online bool) []byte {
+	t.Helper()
+	b, err := delivery.EncodePresence(delivery.Presence{Online: online})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return b
+}
+
+func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3
+	var s PresenceSet
+	if s.Online("ace") {
+		t.Error("a node the watchman never wrote is online")
+	}
+	if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") {
+		t.Error("ace online was not taken in")
+	}
+	if s.Put("ace", presenceValue(t, true), false) {
+		t.Error("the same presence again counted as a change")
+	}
+	if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") {
+		t.Error("ace going offline was not taken in")
+	}
+	s.Put("g14", presenceValue(t, true), false)
+	if !s.Put("g14", nil, true) || s.Online("g14") {
+		t.Error("a deleted key is still online")
+	}
+	// A part's instance (part.<part>.<instance>) is not a node.
+	if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") {
+		t.Error("a part's key was taken for a node")
+	}
+	if s.Put("novox", []byte("junk"), false) {
+		t.Error("junk counted")
+	}
+}

modules/mesh-delivery-builder/internal/build/run.go

@@ -28,7 +28,9 @@ type Builder struct {
 	Claimer    Claimer
 	Runner     Runner
 	Toolchains ToolchainEntries
-	Manifests  Manifests
+	// Presence says which nodes are online (the watchman's); a node that is not is not chosen.
+	Presence  interface{ Online(node string) bool }
+	Manifests Manifests
 	// Nodes is how many build nodes build each fingerprint: two until release R4e (ADR 0478 clause 4).
 	Nodes int
 	// Limit is how many builds run at once; four when 0.
@@ -182,6 +184,14 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 	if read && planErr == nil && len(plan.Tasks) == 0 {
 		return // nothing in the manifest for a builder to build
 	}
+	if planErr == nil {
+		if why := b.stale(m, plan); why != "" {
+			// The fingerprint names a toolchain the build nodes no longer agree on: a new fingerprint is coming, and a build
+			// now would be of this source with another toolchain (ADR 0477 clause 2). Nothing is written.
+			b.logf("%s at %s is not built: %s", module, m.FP2, why)
+			return
+		}
+	}
 	cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest)
 	if err != nil {
 		b.logf("%s: %v", module, err)
@@ -194,6 +204,10 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 	if ctx.Err() != nil {
 		return
 	}
+	if result.State == "" {
+		b.logf("%s: left building, the toolchains moved under it", cl.Key)
+		return
+	}
 	result.By, result.Manifest = b.Claimer.By, m.Manifest
 	if err := b.Claimer.Finish(ctx, cl, result); err != nil {
 		b.logf("%s: the build could not be written: %v", cl.Key, err)
@@ -204,6 +218,23 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
 	}
 }
 
+// stale is why the module's fingerprint is not built now, or empty when it is: the toolchain each language of the plan
+// has agreed (novox/hq ADR 0465) must be the one the fingerprint was computed with (Module.Toolchains).
+func (b *Builder) stale(m delivery.Module, plan Plan) string {
+	for _, bm := range plan.Modules() {
+		language := LanguageOf[bm]
+		tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
+		if none != "" {
+			return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
+		}
+		want, ok := m.Toolchains[language]
+		if !ok || want.Version != tc.Version || want.Recipe != tc.Recipe {
+			return fmt.Sprintf("the fingerprint names %s %s (recipe %s) and the nodes agree on %s (recipe %s)", language, want.Version, want.Recipe, tc.Version, tc.Recipe)
+		}
+	}
+	return ""
+}
+
 func failed(format string, args ...any) delivery.BuilderBuild {
 	return delivery.BuilderBuild{State: delivery.BuildFailed, Error: fmt.Sprintf(format, args...)}
 }
@@ -221,6 +252,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
 	case planErr != nil:
 		return failed("%s", planErr)
 	case m.Path == "":
+		// Empty is not known (a root module's path is "."): the indexer writes it again (issue 1615).
 		return failed("the module's key names no directory (path), so the build tools cannot be told where %s lives", module)
 	}
 	repository := delivery.RepositoryName(m.Repository)
@@ -229,10 +261,13 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
 	var steps []string
 	for _, bm := range plan.Modules() {
 		language := LanguageOf[bm]
-		nodes, tc, err := ChooseNodes(language, b.Toolchains.Entries(language), b.runningOn(bm), b.Nodes)
+		nodes, tc, err := ChooseNodes(language, b.Toolchains.Entries(language), b.runningOn(bm), b.online, b.Nodes)
 		if err != nil {
 			return failed("%s", err)
 		}
+		if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
+			return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
+		}
 		used[language] = delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
 		if len(nodes) == 1 {
 			steps = append(steps, fmt.Sprintf("%s on one node, %s", bm, nodes[0]))
@@ -283,6 +318,8 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
 	return delivery.BuilderBuild{State: delivery.BuildDone, Step: "built " + strings.Join(steps, "; "), Artifacts: artifacts, Toolchains: used}
 }
 
+func (b *Builder) online(node string) bool { return b.Presence == nil || b.Presence.Online(node) }
+
 func (b *Builder) runningOn(buildModule string) map[string]int {
 	b.mu.Lock()
 	defer b.mu.Unlock()

modules/mesh-delivery-builder/internal/build/run_test.go

@@ -95,20 +95,22 @@ type rig struct {
 	st    *memStore
 	bm    *buildModules
 	tools *ToolchainSet
+	pres  *PresenceSet
 	mf    *fakeManifests
 	key   string
 }
 
 func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
 	t.Helper()
-	r := &rig{st: newMem(), bm: newBuildModules(), tools: &ToolchainSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
+	r := &rig{st: newMem(), bm: newBuildModules(), tools: &ToolchainSet{}, pres: &PresenceSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
 	for _, n := range []string{"ace", "novox"} {
 		r.tools.Put(GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false)
 		r.tools.Put(NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false)
+		r.pres.Put(n, presenceValue(t, true), false)
 	}
 	r.b = &Builder{
 		Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second},
-		Toolchains: r.tools, Manifests: r.mf, Nodes: 2, Logf: t.Logf,
+		Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf,
 	}
 	var err error
 	if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
@@ -117,9 +119,14 @@ func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
 	return r
 }
 
-func moduleValue(t *testing.T, path string) []byte {
+// agreedToolchains are what the rig's nodes report, and so what a fingerprint computed now names.
+var agreedToolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}
+
+func moduleValue(t *testing.T, path string) []byte { return moduleValueWith(t, path, agreedToolchains) }
+
+func moduleValueWith(t *testing.T, path string, toolchains map[string]delivery.ToolchainUsed) []byte {
 	t.Helper()
-	data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp1, Manifest: mdigest('a'), Path: path})
+	data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp1, Manifest: mdigest('a'), Path: path, Toolchains: toolchains})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -244,16 +251,6 @@ func TestAModuleKeyWithNoPathFailsNamingThatReason(t *testing.T) { // 1607/AC8,
 	}
 }
 
-func TestNoQualifyingNodeFailsTheBuildNamingTheLanguage(t *testing.T) { // 1607/AC4
-	r := newRig(t, tsBundle)
-	r.tools.Put(NodeBuildModule, "ace.typescript", nil, true)
-	r.tools.Put(NodeBuildModule, "novox.typescript", nil, true)
-	v := r.consider(t, "modules/m")
-	if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "typescript") || !strings.Contains(v.Error, delivery.ToolchainNoReport) {
-		t.Errorf("build = %+v, want failed naming typescript and that no toolchain is reported", v)
-	}
-}
-
 func TestAnUnreadableManifestFailsTheBuild(t *testing.T) {
 	r := newRig(t, goBundle)
 	delete(r.mf.byDigest, mdigest('a'))
@@ -291,7 +288,7 @@ func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t *
 
 func TestTwoInstancesBuildOneFingerprintOnce(t *testing.T) { // 1607/AC7
 	r := newRig(t, goBundle)
-	other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Manifests: r.mf, Nodes: 2, Logf: t.Logf}
+	other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf}
 	value := moduleValue(t, "modules/m")
 	var wg sync.WaitGroup
 	for _, b := range []*Builder{r.b, other} {
@@ -320,3 +317,80 @@ func TestAModulesValueThatArrivesWhileItsKeyIsBeingBuiltIsLookedAtAgainWhenItEnd
 		t.Errorf("build = %+v, want the failed key built again as a new revision after the first build ended", v)
 	}
 }
+
+func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOtherwise(t *testing.T) { // 1615/AC4
+	r := newRig(t, goBundle)
+	old := map[string]delivery.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}}
+	r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old))
+	r.b.Wait()
+	if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
+		t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls))
+	}
+	// A recipe that differs is a different toolchain too, and a value that names none (written before) is not built.
+	r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "0"}}))
+	r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
+	r.b.Wait()
+	if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
+		t.Errorf("keys %v and %d calls for a recipe that differs or a value that names no toolchain", r.st.vals, len(r.bm.calls))
+	}
+	// The toolchain moves under a key that is not built yet: the old fingerprint stays unbuilt, the new one builds.
+	r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false)
+	r.tools.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false)
+	r.b.Reconsider(context.Background())
+	r.b.Wait()
+	if len(r.st.vals) != 0 {
+		t.Errorf("keys %v: the old fingerprint was built with the new toolchain", r.st.vals)
+	}
+	r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}}))
+	r.b.Wait()
+	if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Toolchains["go"].Version != "go1.27.2" {
+		t.Errorf("build = %+v, want done with go1.27.2", v)
+	}
+}
+
+func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4
+	r := newRig(t, goBundle)
+	r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false)
+	r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
+	r.b.Wait()
+	if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
+		t.Errorf("keys %v and %d calls while the nodes disagree", r.st.vals, len(r.bm.calls))
+	}
+}
+
+func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/AC4
+	r := newRig(t, goBundle)
+	if v := r.consider(t, delivery.RootPath); v.State != delivery.BuildDone {
+		t.Fatalf("build = %+v, want done for a root module", v)
+	}
+	if args := r.bm.callsOf("build_go_build")[0].body.(map[string]any); args["module"] != "." {
+		t.Errorf("build args = %+v, want module \".\"", args)
+	}
+}
+
+func TestANodeThatIsNotOnlineIsNotChosenAndComingBackWakesTheBuild(t *testing.T) { // 1607 review 3
+	r := newRig(t, goBundle)
+	r.tools.Put(GoBuildModule, "g14.go", tcValue(t, "go", "go1.27.1"), false) // a laptop that reports its toolchain and sleeps
+	v := r.consider(t, "modules/m")
+	if v.State != delivery.BuildDone {
+		t.Fatalf("build = %+v", v)
+	}
+	for _, c := range r.bm.calls {
+		if c.node == "g14" {
+			t.Errorf("a call went to g14, which is not online: %+v", c)
+		}
+	}
+	// With no node online the key fails naming why; a node coming online builds it as a new revision.
+	r2 := newRig(t, goBundle)
+	r2.pres.Put("ace", nil, true)
+	r2.pres.Put("novox", nil, true)
+	if v := r2.consider(t, "modules/m"); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "online") {
+		t.Fatalf("build = %+v, want failed because no node is online", v)
+	}
+	r2.pres.Put("ace", presenceValue(t, true), false)
+	r2.b.Reconsider(context.Background())
+	r2.b.Wait()
+	if v := valueOf(t, r2.st, r2.key); v.State != delivery.BuildDone {
+		t.Errorf("build = %+v, want done once a node is online", v)
+	}
+}

modules/mesh-delivery-builder/manifest_test.go

@@ -81,7 +81,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 		t.Errorf("state = %+v, want %+v", m.State, want)
 	}
 	if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains",
-		"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
+		"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence"}; !reflect.DeepEqual(sorted(m.Reads), want) {
 		t.Errorf("reads = %v, want %v", m.Reads, want)
 	}
 	if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) {

modules/mesh-delivery-builder/module.json

@@ -15,7 +15,8 @@
     "mesh-build-node.steps",
     "mesh-build-node.toolchains",
     "mesh-delivery-indexer.manifests",
-    "mesh-delivery-indexer.modules"
+    "mesh-delivery-indexer.modules",
+    "mesh-delivery-watchman.presence"
   ],
   "invokes": [
     "mesh-build-go.build_go_build",