Merge pull request 'Builder: announce a done build again, so the controller composes a module whose composition and grant were lost (recovery of novox and ace, 2026-10-29)' (#111) from fix/announce-done-again into main

modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go

@@ -94,6 +94,34 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q
 				return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer,
 					"next": build.StuckTool + " shows the key and its asks while they wait"}, nil
 			}},
+		{Name: build.AnnounceAgainTool,
+			Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " +
+				"again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " +
+				"take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " +
+				"or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool +
+				" builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.",
+			Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{
+				"module": map[string]any{"type": "string", "description": "the module's name"},
+				"fp":     map[string]any{"type": "string", "description": "its new-form fingerprint, src2:<hex>"},
+				"why":    map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}},
+			Run: func(args map[string]any) (any, error) {
+				key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now())
+				if err != nil {
+					return nil, err
+				}
+				raw, err := buildAgain(key, value)
+				if err != nil {
+					return nil, err
+				}
+				r, err := build.DecodeReAsk(raw)
+				if err != nil {
+					return nil, err
+				}
+				if r.Refused || r.Revision == 0 {
+					return nil, errors.New(r.Answer)
+				}
+				return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil
+			}},
 		{Name: build.SeatQueueTool,
 			Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_<LANGUAGE>), as the builder reads it: the stream's message count and " +
 				"sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " +

modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go

@@ -19,6 +19,7 @@ import (
 type askAgainer interface {
 	AskAgain(key, why string) []string
 	BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult
+	AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult
 }
 
 // buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits
@@ -45,6 +46,10 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre
 		go builtAgain(ctx, b, kv, e, r, logf)
 		return
 	}
+	if r.Mode == build.ModeAnnounce {
+		announcedAgain(ctx, b, kv, e, r, logf)
+		return
+	}
 	asked := b.AskAgain(e.Key(), r.Why)
 	if len(asked) == 0 {
 		r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands"
@@ -114,3 +119,22 @@ func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jets
 		logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
 	}
 }
+
+// announcedAgain answers a reasks value of mode announce (the tool builder_announce_again): the done key is written again
+// unchanged, and the value is written back with the new revision, or why nothing was done.
+func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) {
+	res := b.AnnounceAgain(ctx, r.Module, r.FP, r.Why, r.By)
+	if res.Refused != "" {
+		r.Answer, r.Refused = res.Refused, true
+	} else {
+		r.Revision, r.Answer = res.Revision, fmt.Sprintf("announced again: the done key is written again, unchanged, as revision %d", res.Revision)
+	}
+	r.Answered = time.Now().UTC()
+	data, err := json.Marshal(r)
+	if err != nil {
+		return
+	}
+	if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+		logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err)
+	}
+}

modules/mesh-delivery-builder/internal/build/announce.go

@@ -0,0 +1,53 @@
+package build
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+// **A done build is announced again by hand** (recovery of photos and link2pay on novox, 2026-10-29). The controller takes a
+// build in when a write of the build key reaches its take-in consumer, and composes then; a done key is never written again
+// by the builder, so a composition or a grant lost afterwards could not be made again by any tool. AnnounceAgain writes the
+// done key again with its own bytes, as a new revision of the same key: its content, state, artifacts and manifest stay
+// identical, and only the write is new. Who asked and why is in the builder's log and in the reasks answer.
+
+// AnnounceAgain rewrites the done key of module at fp unchanged, as a new revision, and answers the revision. A key that is
+// not done is refused with why: failed (builder_build_again builds it) or waiting (builder_ask_again).
+func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) BuildAgainResult {
+	b.init()
+	key, err := delivery.BuilderBuildKey(module, fp)
+	if err != nil {
+		return BuildAgainResult{Refused: err.Error()}
+	}
+	value, rev, found, err := b.Claimer.Store.Get(ctx, key)
+	if err != nil {
+		return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)}
+	}
+	if !found {
+		return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing to announce", key)}
+	}
+	cur, err := delivery.DecodeBuilderBuild(value)
+	if err != nil {
+		return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)}
+	}
+	switch cur.State {
+	case delivery.BuildDone:
+	case delivery.BuildFailed:
+		return BuildAgainResult{Refused: fmt.Sprintf("%s is failed, not done: %s builds it again", key, BuildAgainTool)}
+	default:
+		return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not done: %s asks its waiting asks again", key, cur.State, AskAgainTool)}
+	}
+	// Compare-and-set at the revision read: a build written meanwhile is not overwritten.
+	nrev, err := b.Claimer.Store.Update(ctx, key, value, rev)
+	if err != nil {
+		if errors.Is(err, ErrConflict) {
+			return BuildAgainResult{Refused: fmt.Sprintf("%s changed while it was announced: read it again", key)}
+		}
+		return BuildAgainResult{Refused: fmt.Sprintf("%s could not be written: %v", key, err)}
+	}
+	b.logf("%s: done, announced again by %s (revision %d): %s", key, by, nrev, why)
+	return BuildAgainResult{Revision: nrev}
+}

modules/mesh-delivery-builder/internal/build/announce_test.go

@@ -0,0 +1,49 @@
+package build
+
+import (
+	"bytes"
+	"context"
+	"strings"
+	"testing"
+
+	"git.novox.be/novox/mesh-sdk/go/delivery"
+)
+
+func TestAnnounceAgainRewritesOnlyADoneKeyUnchanged(t *testing.T) { // recovery of photos and link2pay, 2026-10-29
+	for _, c := range []struct {
+		name   string
+		key    *delivery.BuilderBuild
+		fp     string
+		refuse string
+	}{
+		{name: "a done key is written again, unchanged, as a new revision", key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}},
+		{name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "x"}},
+		{name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked"}},
+		{name: "a key with no build is refused", refuse: "has no build key"},
+		{name: "a fingerprint that is not one is refused", refuse: "fingerprint", fp: "nope"},
+	} {
+		t.Run(c.name, func(t *testing.T) {
+			r := newRig(t, goBundle)
+			var rev uint64
+			if c.key != nil {
+				c.key.Manifest = manifestD
+				rev = putBuild(t, r.st, r.key, *c.key)
+			}
+			before := bytes.Clone(r.st.vals[r.key])
+			fp := fp1
+			if c.fp != "" {
+				fp = c.fp
+			}
+			res := r.b.AnnounceAgain(context.Background(), "m", fp, "grant lost", "shanks.lead")
+			if c.refuse != "" {
+				if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || r.st.revs[r.key] != rev || !bytes.Equal(r.st.vals[r.key], before) {
+					t.Fatalf("answer %+v, key at revision %d (was %d); want refused with %q and nothing written", res, r.st.revs[r.key], rev, c.refuse)
+				}
+				return
+			}
+			if res.Refused != "" || res.Revision <= rev || r.st.revs[r.key] != res.Revision || !bytes.Equal(r.st.vals[r.key], before) {
+				t.Fatalf("answer %+v, key at revision %d (was %d); want a new revision and the same bytes", res, r.st.revs[r.key], rev)
+			}
+		})
+	}
+}

modules/mesh-delivery-builder/internal/build/stuck.go

@@ -24,6 +24,9 @@ const (
 	AskAgainTool = "builder_ask_again"
 	// BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild.
 	BuildAgainTool = "builder_build_again"
+	// AnnounceAgainTool writes a done key again unchanged, so the controller takes it in and composes again: a reasks value
+	// of Mode ModeAnnounce.
+	AnnounceAgainTool = "builder_announce_again"
 	// ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key.
 	ReAsksState  = "reasks"
 	ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState
@@ -56,6 +59,9 @@ type ReAsk struct {
 // ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again.
 const ModeBuild = "build"
 
+// ModeAnnounce is the Mode of a reasks value that asks a done key to be announced again.
+const ModeAnnounce = "announce"
+
 // DecodeReAsk reads a reasks value.
 func DecodeReAsk(data []byte) (ReAsk, error) {
 	var r ReAsk
@@ -94,6 +100,19 @@ func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, er
 	return key, data, err
 }
 
+// NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value.
+func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) {
+	key, err := delivery.BuilderBuildKey(module, fp)
+	if err != nil {
+		return "", nil, err
+	}
+	if strings.TrimSpace(why) == "" {
+		return "", nil, errors.New("say why the done build is announced again (why)")
+	}
+	data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeAnnounce, Module: module, FP: fp})
+	return key, data, err
+}
+
 // StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known).
 type StateRead struct {
 	Keys func() ([]string, error)

modules/mesh-delivery-builder/manifest_test.go

@@ -116,7 +116,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
 		t.Errorf("secrets-owner = %q", m.SecretsOwner)
 	}
 	// 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again.
-	if want := []string{"builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
+	if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) {
 		t.Errorf("tools = %v, want %v", m.Tools, want)
 	}
 	if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) {

modules/mesh-delivery-builder/module.json

@@ -3,6 +3,7 @@
   "version": "1",
   "slug": "builder",
   "tools": [
+    "builder_announce_again",
     "builder_ask_again",
     "builder_build_again",
     "builder_seat_queue",