From 48cd554f46f79dfb12198e39454a0bd4f6808aa7 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 28 Oct 2026 14:25:03 +0100 Subject: [PATCH] Add builder_build_again: build a failed key again by hand as a new revision (issue 1126 note 72) The builder retakes a failed key only when it looks at its module again, which nothing does for an unchanged module. The tool makes it look now, with the claim's own rules, and refuses a key that is done or waiting. --- .../cmd/mesh-delivery-builder-tools/main.go | 70 ++++++-- .../cmd/mesh-delivery-builder/reasks.go | 34 ++++ .../internal/build/buildagain.go | 150 ++++++++++++++++++ .../internal/build/run.go | 14 +- .../internal/build/run_test.go | 56 +++++++ .../internal/build/stuck.go | 27 ++++ .../mesh-delivery-builder/manifest_test.go | 2 +- modules/mesh-delivery-builder/module.json | 1 + 8 files changed, 338 insertions(+), 16 deletions(-) create mode 100644 modules/mesh-delivery-builder/internal/build/buildagain.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 3bb3fd1..25d7794 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -7,6 +7,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "os" "strings" @@ -24,8 +25,8 @@ import ( type queueAsker func(key string, value []byte) (json.RawMessage, error) // Tools are the builder's tools: stuck reads, askAgain writes the reasks value, queue writes a queues value and waits for -// the builder's read. -func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue queueAsker, now func() time.Time) []stdio.Tool { +// the builder's read, buildAgain writes a reasks value of mode build and waits for the builder's answer. +func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, queue, buildAgain queueAsker, now func() time.Time) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: build.StuckTool, @@ -57,6 +58,36 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q return map[string]any{"key": key, "asked": true, "next": "the builder answers in " + build.ReAsksState + "." + key + "; " + build.StuckTool + " shows the answer and the new ask"}, nil }}, + {Name: build.BuildAgainTool, + Description: "Build a failed key again (rebuild a module, build a failed key again): the builder writes the key of module and " + + "fingerprint building as a new revision of the same key and asks the build seats, as it does by itself when it looks at a module " + + "again (ADR 0429 clause 18). Answers the new revision and the ask id of each build module (go, node, image). Refuses a key that " + + "is not failed, and says why: done (nothing to do), or waiting (building or queued: " + build.AskAgainTool + " asks its waiting asks " + + "again); a key failed for a reason the builder never retries on the same fingerprint (a pin missing, a version taken, digests that " + + "differ) and a fingerprint its module no longer has are refused too. Use it for a key builder_stuck does not show, which failed.", + Input: map[string]any{"type": "object", "required": []string{"module", "fp", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module's name"}, + "fp": map[string]any{"type": "string", "description": "its new-form fingerprint, src2:"}, + "why": map[string]any{"type": "string", "description": "why it is built again, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewBuildAgain(str(args, "module"), str(args, "fp"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + raw, err := buildAgain(key, value) + if err != nil { + return nil, err + } + r, err := build.DecodeReAsk(raw) + if err != nil { + return nil, err + } + if r.Refused { + return nil, errors.New(r.Answer) + } + return map[string]any{"key": key, "revision": r.Revision, "asks": r.Asks, "answer": r.Answer, + "next": build.StuckTool + " shows the key and its asks while they wait"}, nil + }}, {Name: build.SeatQueueTool, Description: "What waits on a build seat's stream (SEAT_NODE_BUILD_), as the builder reads it: the stream's message count and " + "sequences, and per kind (delivery, check, recheck) its messages oldest first, at most limit each, with sequence, ask id, when asked " + @@ -87,8 +118,17 @@ func Tools(r build.StuckReader, askAgain func(key string, value []byte) error, q // askQueue writes the queues value at key and waits, on a watch of the key, at most bound for the builder's answer. func askQueue(s stdio.KeptState, bound time.Duration) queueAsker { + return askKept(s, build.QueuesState, build.QueuesBucket, "the read", bound, func(v []byte) bool { + a, err := build.DecodeQueueAsk(v) + return err == nil && !a.Answered.IsZero() + }) +} + +// askKept writes a value at key of a state the builder watches and answers there, and waits, on a watch of the key, at most +// bound for a value written after it that answered says is the builder's answer. +func askKept(s stdio.KeptState, state, bucket, what string, bound time.Duration, answered func([]byte) bool) queueAsker { return func(key string, value []byte) (json.RawMessage, error) { - answered := make(chan json.RawMessage, 1) + heard := make(chan json.RawMessage, 1) var rev uint64 var mu sync.Mutex if err := s.Watch(key, func(c stdio.StateChange) error { @@ -98,36 +138,34 @@ func askQueue(s stdio.KeptState, bound time.Duration) queueAsker { if c.Op != "put" || mine == 0 || c.Revision <= mine { return nil } - if a, err := build.DecodeQueueAsk(c.Value); err == nil && !a.Answered.IsZero() { + if answered(c.Value) { select { - case answered <- c.Value: + case heard <- c.Value: default: } } return nil }); err != nil { - return nil, fmt.Errorf("watching %s.%s: %w", build.QueuesState, key, err) + return nil, fmt.Errorf("watching %s.%s: %w", state, key, err) } r, err := s.Put(key, json.RawMessage(value)) if err != nil { - return nil, fmt.Errorf("the read could not be written to %s: %w", build.QueuesBucket, err) + return nil, fmt.Errorf("%s could not be written to %s: %w", what, bucket, err) } mu.Lock() rev = r mu.Unlock() // The watch may have heard the answer before the revision was known: read it once too. - if e, err := s.Get(key); err == nil && e != nil && e.Revision > r { - if a, err := build.DecodeQueueAsk(e.Value); err == nil && !a.Answered.IsZero() { - return e.Value, nil - } + if e, err := s.Get(key); err == nil && e != nil && e.Revision > r && answered(e.Value) { + return e.Value, nil } wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8) defer cancel() select { - case v := <-answered: + case v := <-heard: return v, nil case <-wait.Done(): - return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", build.QueuesState, key, bound, build.QueuesBucket) + return nil, fmt.Errorf("the builder did not answer %s.%s within %s: it is not running, or cannot read %s", state, key, bound, bucket) } } } @@ -174,7 +212,11 @@ func main() { _, err := reasks.Put(key, json.RawMessage(value)) return err } - if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), time.Now)); err != nil { + buildAgain := askKept(reasks, build.ReAsksState, build.ReAsksBucket, "the request", 25*time.Second, func(v []byte) bool { + a, err := build.DecodeReAsk(v) + return err == nil && !a.Answered.IsZero() + }) + if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, time.Now)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index de72b5c..70d8b00 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -18,8 +18,13 @@ import ( // askAgainer is the part of the builder a reasks value asks through. type askAgainer interface { AskAgain(key, why string) []string + BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) build.BuildAgainResult } +// buildAgainBound is how long the builder waits for the claim and the asks of a key built again by hand; the tool waits +// longer for the builder's answer. +const buildAgainBound = 15 * time.Second + // reAsked answers one value of the reasks state (the tool builder_ask_again, issue 1945): a value not yet answered asks the // key's waiting lanes again and is written back, by compare-and-set at its revision, with what the builder did. A value // answered already (this builder's own write) or deleted is left. @@ -35,6 +40,11 @@ func reAsked(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstre if r.Answer != "" { return } + if r.Mode == build.ModeBuild { + // The answer comes when the claim and the asks are written, so it is made beside the watch, not in it. + go builtAgain(ctx, b, kv, e, r, logf) + return + } asked := b.AskAgain(e.Key(), r.Why) if len(asked) == 0 { r.Answer = "no ask of this key waits in this builder now (it is not building, or its build has not reached its ask yet): nothing was asked; builder_stuck shows where it stands" @@ -74,3 +84,27 @@ func emitAskedAgain(ctx context.Context, js jetstream.JetStream, cred process.Ac } return nil } + +// builtAgain answers a reasks value of mode build (the tool builder_build_again, issue 1126 note 72): the failed key is +// built again as a new revision, and the value is written back with the revision and the asks put, or why nothing was done. +func builtAgain(ctx context.Context, b askAgainer, kv jetstream.KeyValue, e jetstream.KeyValueEntry, r build.ReAsk, logf func(string, ...any)) { + res := b.BuildAgain(ctx, r.Module, r.FP, r.Why, r.By, buildAgainBound) + switch { + case res.Refused != "": + r.Answer, r.Refused = res.Refused, true + default: + r.Answer, r.Revision, r.Asks = "built again: the key is written building as revision "+fmt.Sprint(res.Revision)+" and its asks are put", res.Revision, res.Asks + if res.Pending != "" { + r.Answer += "; " + res.Pending + } + } + r.Answered = time.Now().UTC() + logf("%s: build again asked by %s (%s): %s", e.Key(), r.By, r.Why, r.Answer) + data, err := json.Marshal(r) + if err != nil { + return + } + if _, err := kv.Update(ctx, e.Key(), data, e.Revision()); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + logf("%s.%s: the answer could not be written: %v", build.ReAsksState, e.Key(), err) + } +} diff --git a/modules/mesh-delivery-builder/internal/build/buildagain.go b/modules/mesh-delivery-builder/internal/build/buildagain.go new file mode 100644 index 0000000..0ae3f35 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/buildagain.go @@ -0,0 +1,150 @@ +package build + +import ( + "context" + "fmt" + "maps" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" +) + +// **A failed key is built again by hand** (novox/hq issue 1126 note 72; ADR 0429 clause 18: "a fingerprint is built again +// only when its build key failed, as a new revision of the same key"). The builder does it by itself whenever it looks at a +// module again (a modules value, a change of the toolchains, artifacts-gone, a moved trunk): Claimer.Claim writes a failed +// key building as a new revision and the build asks the build seats. Nothing looks at an unchanged module on its own, so a key +// that failed (a node refused the build, a toolchain was missing) stayed failed until the module changed. The tool +// builder_build_again (BuildAgain) makes the builder look at one module now, which does exactly that and no more: it keeps +// the claim's rules, so a key failed for a reason the builder never retries on the same fingerprint stays refused. + +// BuildAgainResult is the builder's answer to a request to build a key again. +type BuildAgainResult struct { + // Refused is why nothing was done, "" when the key was written building as a new revision. + Refused string + // Revision is the revision the claim wrote; Asks are the ask ids put by language, as far as they were put within the bound. + Revision uint64 + Asks map[string]string + // Pending says the builder had not put every ask when it answered. + Pending string +} + +// handWatch follows one request to build a key again until its asks are put. +type handWatch struct { + rev uint64 + lanes int + asks map[string]string + done chan struct{} + shut bool +} + +// settle closes done once the claim is written and every lane has put its ask. Call with b.mu held. +func (w *handWatch) settle() { + if !w.shut && w.rev > 0 && w.lanes >= 0 && len(w.asks) >= w.lanes { + w.shut = true + close(w.done) + } +} + +func (b *Builder) hand(key string, f func(w *handWatch)) { + b.mu.Lock() + defer b.mu.Unlock() + if w := b.hands[key]; w != nil { + f(w) + } +} + +func (b *Builder) handClaimed(key string, rev uint64) { + b.hand(key, func(w *handWatch) { w.rev = rev; w.settle() }) +} + +func (b *Builder) handLanes(key string, n int) { + b.hand(key, func(w *handWatch) { w.lanes = n; w.settle() }) +} + +func (b *Builder) handAsk(key, language, id string) { + b.hand(key, func(w *handWatch) { w.asks[language] = id; w.settle() }) +} + +// handEnded ends the watch of a request whose look ended: it wrote no claim, or the build ended. +func (b *Builder) handEnded(key string) { + b.hand(key, func(w *handWatch) { + if !w.shut { + w.shut = true + close(w.done) + } + }) +} + +// BuildAgain builds the failed key of module at fp again, as the builder does by itself: the key is written building as a +// new revision and the build module asks are put on the build seats. It answers once the asks are put, or at bound. A key +// that is not failed is refused with why: done (nothing to do) or waiting (builder_ask_again asks its waiting asks again). +// ctx is the builder's own context: the build it starts lives on it. +func (b *Builder) BuildAgain(ctx context.Context, module, fp, why, by string, bound time.Duration) BuildAgainResult { + b.init() + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return BuildAgainResult{Refused: err.Error()} + } + value, _, found, err := b.Claimer.Store.Get(ctx, key) + if err != nil { + return BuildAgainResult{Refused: fmt.Sprintf("%s cannot be read: %v", key, err)} + } + if !found { + return BuildAgainResult{Refused: fmt.Sprintf("%s has no build key: nothing failed. The builder builds a fingerprint by itself once the indexer writes it to its module", key)} + } + cur, err := delivery.DecodeBuilderBuild(value) + if err != nil { + return BuildAgainResult{Refused: fmt.Sprintf("%s holds what is not a build: %v", key, err)} + } + switch { + case cur.State == delivery.BuildDone: + return BuildAgainResult{Refused: fmt.Sprintf("%s is done: nothing to do", key)} + case cur.State == delivery.BuildBuilding || cur.State == delivery.BuildQueued: + return BuildAgainResult{Refused: fmt.Sprintf("%s is waiting (%s: %s): %s asks its waiting asks again", key, cur.State, cur.Step, AskAgainTool)} + case cur.State != delivery.BuildFailed: + return BuildAgainResult{Refused: fmt.Sprintf("%s is %q, not failed", key, cur.State)} + case delivery.FailedUntilChanged(cur.Error) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix): + return BuildAgainResult{Refused: fmt.Sprintf("%s failed for a reason the builder does not retry on the same fingerprint, it waits for another fingerprint: %s", key, cur.Error)} + } + b.mu.Lock() + m, known := b.known[module] + busy := b.inflight[key] + b.mu.Unlock() + switch { + case !known || m.FP2 != fp: + return BuildAgainResult{Refused: fmt.Sprintf("module %s does not have the fingerprint %s now (a newer one superseded it, or its modules value is not read yet): the builder builds the module's current fingerprint", module, fp)} + case busy: + return BuildAgainResult{Refused: fmt.Sprintf("%s is being looked at by this builder now: wait for it to end", key)} + } + w := &handWatch{lanes: -1, asks: map[string]string{}, done: make(chan struct{})} + b.mu.Lock() + b.hands[key] = w + b.mu.Unlock() + defer func() { + b.mu.Lock() + delete(b.hands, key) + b.mu.Unlock() + }() + b.logf("%s: failed (%s); built again by hand, asked by %s: %s", key, cur.Error, by, why) + b.consider(ctx, module, m) + wait, stop := context.WithTimeout(ctx, bound) // a context's bound for a person's answer (ADR 0437 §8) + defer stop() + select { + case <-w.done: + case <-wait.Done(): + } + b.mu.Lock() + defer b.mu.Unlock() + res := BuildAgainResult{Revision: w.rev, Asks: maps.Clone(w.asks)} + switch { + case w.rev == 0 && w.shut: + res.Refused = fmt.Sprintf("the builder looked at %s and wrote nothing (a toolchain moved, the claim was lost or could not be written): its log says why", key) + case w.rev == 0: + res.Refused = fmt.Sprintf("the builder has not claimed %s within %s: it waits behind other builds; builder_stuck shows where it stands", key, bound) + case !w.shut: + res.Pending = fmt.Sprintf("the key is building at revision %d, and not every ask was put within %s; builder_stuck shows the asks", w.rev, bound) + } + return res +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 6283da8..45fa6d5 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -89,6 +89,8 @@ type Builder struct { held map[string]bool // waiting are the lanes that wait for an ask's step, by build key and language: AskAgain asks through them. waiting map[string]map[string]chan string + // hands are the requests to build a key again by hand that wait for the claim and its asks (BuildAgain), by build key. + hands map[string]*handWatch } type planned struct { @@ -106,6 +108,7 @@ func (b *Builder) init() { b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} b.published, b.held, b.waiting = map[string]bool{}, map[string]bool{}, map[string]map[string]chan string{} + b.hands = map[string]*handWatch{} }) } @@ -386,6 +389,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { } } key, _ := delivery.BuilderBuildKey(module, m.FP2) + defer b.handEnded(key) b.mu.Lock() trunk := b.trunkFP[key] first := !b.tried[key] @@ -400,6 +404,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if cl == nil { return } + b.handClaimed(key, cl.Rev) if cl.Unasked != nil { b.logf("%v", cl.Unasked) } @@ -535,6 +540,9 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p // One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3). lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}}) } + if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil { + b.handLanes(key, len(lanes)) + } var wg sync.WaitGroup for _, l := range lanes { wg.Add(1) @@ -717,7 +725,11 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error { v.Asks = map[string]string{} } v.Asks[language] = id - return n.b.Claimer.Finish(ctx, n.cl, v) + if err := n.b.Claimer.Finish(ctx, n.cl, v); err != nil { + return err + } + n.b.handAsk(n.cl.Key, language, id) + return nil } // runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index c8fcd3e..e2e822d 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -694,7 +694,25 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { puts int done bool asking int // the asking the event says, 0 when none is said + // again, when set, is a request to build the key again by hand (BuildAgain, issue 1126 note 72) of the key as + // it stands (key, absent when nil), the module known at known's fingerprint ("" for the rig's): refuse is a part of why it + // is refused, "" when it is built again as a new revision and its ask id answered. + again bool + key *delivery.BuilderBuild + known string + refuse string }{ + {name: "build again: a failed key is written building as a new revision and asks the seat", again: true, + key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: the step failed: no tun"}}, + {name: "build again: a done key is refused, nothing to do", again: true, refuse: "is done: nothing to do", + key: &delivery.BuilderBuild{State: delivery.BuildDone, Step: "built"}}, + {name: "build again: a building key is refused, builder_ask_again is for it", again: true, refuse: "is waiting", + key: &delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: "mvl0frv18d1960"}}}, + {name: "build again: a key failed for a pin missing is refused, it waits for another fingerprint", again: true, refuse: "does not retry", + key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: delivery.NotPinned + "web"}}, + {name: "build again: a key with no build is refused", again: true, refuse: "has no build key"}, + {name: "build again: a fingerprint its module no longer has is refused", again: true, refuse: "does not have the fingerprint", known: "src2:" + strings.Repeat("9", 64), + key: &delivery.BuilderBuild{State: delivery.BuildFailed, Error: "ace: no tun"}}, {name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2}, {name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2}, {name: "given up into DEAD_LETTERS: its wait ends failed with the words and it is asked again at once", dead: true, puts: 2, done: true, asking: 2}, @@ -703,6 +721,44 @@ func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { } { t.Run(c.name, func(t *testing.T) { // 1945 r := newRig(t, goBundle) + if c.again { + var rev uint64 + if c.key != nil { + c.key.Manifest = manifestD + rev = putBuild(t, r.st, r.key, *c.key) + } + m, err := delivery.DecodeModule(moduleValue(t, "modules/m")) + if err != nil { + t.Fatal(err) + } + if c.known != "" { + m.FP2 = c.known + } + r.b.init() + r.b.known["m"] = m // the builder has read the modules value, and looks at nothing until asked + log := &logged{} + r.b.Logf = log.logf + res := r.b.BuildAgain(context.Background(), "m", fp1, "no tun on ace", "shanks.lead", 2*time.Second) + r.b.Wait() + after := r.st.revs[r.key] + if c.refuse != "" { + if !strings.Contains(res.Refused, c.refuse) || res.Revision != 0 || len(r.asks.Asks()) != 0 || after != rev { + t.Fatalf("refused %q (revision %d, %d asks, key at revision %d was %d); want refused with %q and nothing written or asked", res.Refused, res.Revision, len(r.asks.Asks()), after, rev, c.refuse) + } + return + } + asks := r.asks.Asks() + if res.Refused != "" || res.Pending != "" || res.Revision <= rev || len(asks) != 1 || res.Asks[delivery.AskGo] != asks[0].ID { + t.Fatalf("answer %+v after %d asks; want the new revision (was %d) and the ask id of the one ask", res, len(asks), rev) + } + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || after <= res.Revision { + t.Errorf("the key = %+v at revision %d; want built from the new revision %d", v, after, res.Revision) + } + if !log.has(r.key, "built again by hand", "no tun on ace", "shanks.lead") { + t.Errorf("the why is not logged: %v", log.lines) + } + return + } r.asks.Answer = nil r.b.Askings = c.askings var said []AskedAgain diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index 50ed667..66d4cc3 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -22,6 +22,8 @@ import ( const ( StuckTool = "builder_stuck" AskAgainTool = "builder_ask_again" + // BuildAgainTool builds a failed key again by hand (issue 1126 note 72): it writes a reasks value of Mode ModeBuild. + BuildAgainTool = "builder_build_again" // ReAsksState is the builder's state of asks to ask again: the bucket mesh-delivery-builder_reasks, keyed as the build key. ReAsksState = "reasks" ReAsksBucket = delivery.BuilderModule + "_" + ReAsksState @@ -36,8 +38,20 @@ type ReAsk struct { At time.Time `json:"at"` Answer string `json:"answer,omitempty"` Answered time.Time `json:"answered,omitzero"` + // Mode is ModeBuild for a request to build a failed key again (BuildAgainTool); empty for an ask asked again. + Mode string `json:"mode,omitempty"` + Module string `json:"module,omitempty"` + FP string `json:"fp,omitempty"` + // Refused, Revision and Asks are the builder's answer to a ModeBuild request: why it did nothing, or the key's new + // revision (building) and the ask ids it put by language. + Refused bool `json:"refused,omitempty"` + Revision uint64 `json:"revision,omitempty"` + Asks map[string]string `json:"asks,omitempty"` } +// ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again. +const ModeBuild = "build" + // DecodeReAsk reads a reasks value. func DecodeReAsk(data []byte) (ReAsk, error) { var r ReAsk @@ -63,6 +77,19 @@ func NewReAsk(module, fp, why, by string, at time.Time) (string, []byte, error) return key, data, err } +// NewBuildAgain is the value builder_build_again writes for a module's fingerprint: its key and the value. +func NewBuildAgain(module, fp, why, by string, at time.Time) (string, []byte, error) { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return "", nil, err + } + if strings.TrimSpace(why) == "" { + return "", nil, errors.New("say why the key is built again (why)") + } + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeBuild, Module: module, FP: fp}) + return key, data, err +} + // StateRead is a state the tools read: its keys, and a key's value with when it was written (zero when not known). type StateRead struct { Keys func() ([]string, error) diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 6679a16..896cf0c 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -116,7 +116,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. - if want := []string{"builder_ask_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"builder_ask_again", "builder_build_again", "builder_seat_queue", "builder_stuck"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } if want := []string{"build.asked-again"}; !reflect.DeepEqual(m.Emits, want) { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index e4d2859..f1b273f 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -4,6 +4,7 @@ "slug": "builder", "tools": [ "builder_ask_again", + "builder_build_again", "builder_seat_queue", "builder_stuck" ],