diff --git a/PR-DESCRIPTION-DRAFT.md b/PR-DESCRIPTION-DRAFT.md new file mode 100644 index 0000000..6fb66cb --- /dev/null +++ b/PR-DESCRIPTION-DRAFT.md @@ -0,0 +1,15 @@ +Builder tool builder_announce_again: announce a done build again + +Recovery of photos and link2pay on novox, 2026-10-29. The controller took both builds in on 10-28 and later lost the composition +and the grant; its take-in composes only when a write of the build key arrives, and no tool wrote a done key again +(builder_build_again refuses done keys). Every want for them was refused. + +Quick fix (this change, about an hour): builder_announce_again (module, fp, why) rewrites a done key with its own bytes as a +new revision of the same key (content, artifacts, manifest identical), by compare-and-set; refuses a key that is not done; logs +who and why. It goes through the reasks state like builder_build_again. +Thorough fix (not here): ADR 0547 and design 47, the node agent composes its own and the assigner stops waiting on controller +compositions, so a lost composition cannot block a want. Chosen quick because the recovery blocks two live modules; the thorough +fix is queued under that ADR. + +Tests: one table-driven unit test (5 cases) in internal/build/announce_test.go; manifest test lists the tool. +Reviewed-by: (to be filled from the reviewer) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 0b093ad..dd9835d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -94,6 +94,34 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer, "next": build.StuckTool + " shows the key and its asks while they wait"}, nil }}, + {Name: build.AnnounceAgainTool, + Description: "Announce a done build again (take in again, compose again): the builder writes the done key of module and fingerprint " + + "again, unchanged, as a new revision of the same key (its content, artifacts and manifest stay identical), so the controller's " + + "take-in hears it and composes the module and writes its grant again. Use it when a done build was taken in but its composition " + + "or grant is gone and every want is refused for it. Answers the new revision. Refuses a key that is not done: " + build.BuildAgainTool + + " builds a failed key again, " + build.AskAgainTool + " asks a waiting key's asks again.", + Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module's name"}, + "fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:"}, + "why": map[string]any{"type": "string", "description": "why it is announced again, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewAnnounceAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + raw, err := buildAgain(key, value) + if err != nil { + return nil, err + } + r, err := build.DecodeReAsk(raw) + if err != nil { + return nil, err + } + if r.Refused || r.Revision == 0 { + return nil, errors.New(r.Answer) + } + return map[string]any{"key": key, "revision": r.Revision, "answer": r.Answer}, nil + }}, {Name: build.SeatQueueTool, Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_), as the builder reads it: the stream's message count and " + "sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " + diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 04836a7..311ba12 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -19,6 +19,7 @@ import ( type askAgainer interface { AskAgain(key, why string) []string BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult + AnnounceAgain(ctx context.Context, module, fp, why, by string) build.BuildAgainResult } // buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits @@ -45,6 +46,10 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre go builtAgain(ctx, b, kv, e, r, logf) return } + if r.Mode == build.ModeAnnounce { + announcedAgain(ctx, b, kv, e, r, logf) + return + } asked := b.AskAgain(e.Key(), r.Why) if len(asked) == 0 { r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands" @@ -114,3 +119,22 @@ func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jets logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err) } } + +// announcedAgain answers a reasks value of mode announce (the tool builder_announce_again): the done key is written again +// unchanged, and the value is written back with the new revision, or why nothing was done. +func announcedAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) { + res := b.AnnounceAgain(ctx, r.Module, r.FP, r.Why, r.By) + if res.Refused != "" { + r.Answer, r.Refused = res.Refused, true + } else { + r.Revision, r.Answer = res.Revision, fmt.Sprintf("announced again: the done key is written again, unchanged, as revision %d", res.Revision) + } + r.Answered = time.Now().UTC() + data, err := json.Marshal(r) + if err != nil { + return + } + if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/announce.go b/modules/mesh-delivery-builder/internal/build/announce.go new file mode 100644 index 0000000..7ccdd9b --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/announce.go @@ -0,0 +1,53 @@ +package build + +import ( + "context" + "errors" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// **A done build is announced again by hand** (recovery of photos and link2pay on novox, 2026-10-29). The controller takes a +// build in when a write of the build key reaches its take-in consumer, and composes then; a done key is never written again +// by the builder, so a composition or a grant lost afterwards could not be made again by any tool. AnnounceAgain writes the +// done key again with its own bytes, as a new revision of the same key: its content, state, artifacts and manifest stay +// identical, and only the write is new. Who asked and why is in the builder's log and in the reasks answer. + +// AnnounceAgain rewrites the done key of module at fp unchanged, as a new revision, and answers the revision. A key that is +// not done is refused with why: failed (builder_build_again builds it) or waiting (builder_ask_again). +func (b *Builder) AnnounceAgain(ctx context.Context, module, fp, why, by string) BuildAgainResult { + b.init() + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return BuildAgainResult{Refused: err.Error()} + } + value, rev, found, err := b.Claimer.Store.Get(ctx, key) + if err != nil { + return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)} + } + if !found { + return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing to announce", key)} + } + cur, err := delivery.DecodeBuilderBuild(value) + if err != nil { + return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)} + } + switch cur.State { + case delivery.BuildDone: + case delivery.BuildFailed: + return BuildAgainResult{Refused: fmt.Sprintf("%s is failed, not done: %s builds it again", key, BuildAgainTool)} + default: + return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not done: %s asks its waiting asks again", key, cur.State, AskAgainTool)} + } + // Compare-and-set at the revision read: a build written meanwhile is not overwritten. + nrev, err := b.Claimer.Store.Update(ctx, key, value, rev) + if err != nil { + if errors.Is(err, ErrConflict) { + return BuildAgainResult{Refused: fmt.Sprintf("%s changed while it was announced: read it again", key)} + } + return BuildAgainResult{Refused: fmt.Sprintf("%s could not be written: %v", key, err)} + } + b.logf("%s: done, announced again by %s (revision %d): %s", key, by, nrev, why) + return BuildAgainResult{Revision: nrev} +} diff --git a/modules/mesh-delivery-builder/internal/build/announce_test.go b/modules/mesh-delivery-builder/internal/build/announce_test.go new file mode 100644 index 0000000..021c900 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/announce_test.go @@ -0,0 +1,49 @@ +package build + +import ( + "bytes" + "context" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func TestAnnounceAgainRewritesOnlyADoneKeyUnchanged(t *testing.T) { // recovery of photos and link2pay, 2026-10-29 + for _, c := range []struct { + name string + key *delivery.BuilderBuild + fp string + refuse string + }{ + {name: "a done key is written again, unchanged, as a new revision", key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}}, + {name: "a failed key is refused, builder_build_again is for it", refuse: "builder_build_again", key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "x"}}, + {name: "a building key is refused, builder_ask_again is for it", refuse: "builder_ask_again", key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked"}}, + {name: "a key with no build is refused", refuse: "has no build key"}, + {name: "a fingerprint that is not one is refused", refuse: "fingerprint", fp: "nope"}, + } { + t.Run(c.name, func(t *testing.T) { + r := newRig(t, goBundle) + var rev uint64 + if c.key != nil { + c.key.Manifest = manifestD + rev = putBuild(t, r.st, r.key, *c.key) + } + before := bytes.Clone(r.st.vals[r.key]) + fp := fp1 + if c.fp != "" { + fp = c.fp + } + res := r.b.AnnounceAgain(context.Background(), "m", fp, "grant lost", "shanks.lead") + if c.refuse != "" { + if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || r.st.revs[r.key] != rev || !bytes.Equal(r.st.vals[r.key], before) { + t.Fatalf("answer %+v, key at revision %d (was %d); want refused with %q and nothing written", res, r.st.revs[r.key], rev, c.refuse) + } + return + } + if res.Refused != "" || res.Revision <= rev || r.st.revs[r.key] != res.Revision || !bytes.Equal(r.st.vals[r.key], before) { + t.Fatalf("answer %+v, key at revision %d (was %d); want a new revision and the same bytes", res, r.st.revs[r.key], rev) + } + }) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 66ba529..72528fe 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -24,6 +24,9 @@ const ( AskAgainTool = "builder_ask_again" // BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild. BuildAgainTool = "builder_build_again" + // AnnounceAgainTool writes a done key again unchanged, so the controller takes it in and composes again: a reasks value + // of Mode ModeAnnounce. + AnnounceAgainTool = "builder_announce_again" // ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key. ReAsksState = "reasks" ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState @@ -56,6 +59,9 @@ type ReAsk struct { // ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again. const ModeBuild = "build" +// ModeAnnounce is the Mode of a reasks value that asks a done key to be announced again. +const ModeAnnounce = "announce" + // DecodeReAsk reads a reasks value. func DecodeReAsk(data []byte) (ReAsk, error) { var r ReAsk @@ -94,6 +100,19 @@ func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, er return key, data, err } +// NewAnnounceAgain is the value builder_announce_again writes for a module's fingerprint: its key and the value. +func NewAnnounceAgain(module, fp, why, by string, at time.Time) (string, []byte, error) { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return "", nil, err + } + if strings.TrimSpace(why) == "" { + return "", nil, errors.New("say why the done build is announced again (why)") + } + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeAnnounce, Module: module, FP: fp}) + return key, data, err +} + // StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known). type StateRead struct { Keys func() ([]string, error) diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 896cf0c..44c3002 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -116,7 +116,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. - if want := []string{"builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index f1b273f..2929850 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -3,6 +3,7 @@ "version": "1", "slug": "builder", "tools": [ + "builder_announce_again", "builder_ask_again", "builder_build_again", "builder_seat_queue",