diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go index 550bfb4fb..251610e5e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/asks_tools.go @@ -129,3 +129,34 @@ func asksTools(waiting func(string) (stdio.Waiting, error), taken takenReader, a }}, } } + +// keepNowTool is the tool assigner_keep_now (design 47 H2 §813 (2); H-A-417): written in the hand-act log first, then the seat +// mesh-wants' act keep-now runs the keeping of the assignment graph's revisions at once in the assigner's process. +const keepNowTool = "assigner_keep_now" + +func keepNowTools(ask func(string, any) (json.RawMessage, error)) []stdio.Tool { + return []stdio.Tool{{Name: keepNowTool, Description: "Keep the assignment graph's revisions now (design 47 H2 §813 (2)): written in the " + + "hand-act log first, then the assigner keeps the pass's newest computed revision at once and answers the revision kept, or " + + "the fault the keeping met (a failed keeping runs again after keeping-retry; keeping-failed is raised at the third in a row).", + Input: schema(map[string]string{"why": "why the keeping runs now"}, []string{"why"}), + Run: func(args map[string]any) (any, error) { + why := strings.TrimSpace(arg(args, "why")) + if why == "" { + return nil, errors.New("say why the keeping runs now") + } + taken, err := meshhandactsclient.Recorder{Ask: ask}.Record(meshhandactsclient.Request{What: keepNowTool, Why: why + " (" + stdio.CallerOf(args) + ")", Cause: "keep-now"}) + if err != nil { + return nil, fmt.Errorf("the hand act was not recorded, so nothing was kept: %w", err) + } + raw, err := ask("seat:"+meshwants.Seat+"."+meshwants.VerbKeepNow, meshwants.KeepNowInput{Why: why}) + if err != nil { + return nil, err + } + var out map[string]any + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("%s answered what is not JSON: %w", meshwants.VerbKeepNow, err) + } + out["hand-act"] = taken.Log + return out, nil + }}} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go index b60771eb2..1eb4cc98e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go @@ -5,18 +5,25 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "slices" "strings" "sync" "time" - mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + mcatc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-assigner/internal/assign" "mesh-delivery-assigner/internal/assigngraph" + "mesh-delivery-assigner/internal/assignment" ) // graphKeeper is the pass's assignment graph (design 47 G §292, A-67): per catalog revision taken in it computes the revision @@ -28,7 +35,7 @@ type graphKeeper struct { // reads is the same store through a handle of the pass's own, for the pass's read of its fill (graphfill.go); nil reads objects. reads assigngraph.Objects // halves is every followed branch's catalog graph revision held now. - halves func() map[string]mcc.Half + halves func() map[string]mcatc.Half // mu guards what the keeping writes and the pass reads: the revision kept, full, and the need handed over (design 47 H-A-259). mu sync.Mutex // handedRev is the revision last handed to the keeping, so a pass whose inputs did not change hands nothing; handed is false @@ -46,14 +53,40 @@ type graphKeeper struct { full, started bool // revisions removes a revision that ceased to be current (revisions.go, H-A-86); nil keeps every revision. revisions *revisionRemover - // computed is the revision computed by the last pass, with its entries: its edges. - computed assigngraph.Computed + // computed is the revision computed by the last pass, with its entries: its edges; hasComputed says a pass computed one. + computed assigngraph.Computed + hasComputed bool + // keepMu runs one keeping at a time, the pass's (keepOne) or one by hand (keepNow). + keepMu sync.Mutex // The keeping off the pass loop (H-A-259): need is what waits to be kept, keeping says one runs; background runs it (nil: at // once, in the caller), and wake makes a pass once a keeping changed what the pass reads. need *keepNeed keeping bool background func(func()) wake func() + // A keeping that fails runs again after retry, keeping-retry, by after (time.AfterFunc; a test hands its own); failures + // counts the failures in a row, and keeping-failed is raised at the third (design 47 H2 §813 (2)). + retry time.Duration + after func(d time.Duration, f func()) + failures int +} + +// The keeping's setting and finding (design 47 H2 §813 (2); ADR 0682 §11: the holder's own deadline). +const ( + keepingRetrySetting = "keeping-retry" + defaultKeepingRetry = 30 * time.Second + kindKeepingFailed = "keeping-failed" + // keepingFailedAt is the failures in a row that raise keeping-failed. + keepingFailedAt = 3 +) + +// keepingRetry reads the setting keeping-retry (§781). +func keepingRetry() (time.Duration, error) { + d, err := stdio.SettingDuration(keepingRetrySetting, defaultKeepingRetry) + if err != nil { + return 0, errors.New("the assigner's " + err.Error()) + } + return d, nil } // keepNeed is one need of the keeping: a computed revision to keep (nil: none), and whether the revisions not current are judged. @@ -99,7 +132,9 @@ func (p *passer) keepGraph(ctx context.Context) string { return "" } c := assigngraph.Compute(p.graphInputs()) - g.computed = c + g.mu.Lock() + g.computed, g.hasComputed = c, true + g.mu.Unlock() for _, u := range c.Uncomputed { if g.said != u { g.said = u @@ -177,6 +212,8 @@ func (p *passer) needKeeping(ctx context.Context, n keepNeed) { // revisions that ceased to be current; it says its duration in the journal, and wakes a pass when it changed what the pass reads. func (p *passer) keepOne(ctx context.Context, n keepNeed) { g := p.graph + g.keepMu.Lock() + defer g.keepMu.Unlock() start := time.Now() changed := false defer func() { @@ -199,6 +236,12 @@ func (p *passer) keepOne(ctx context.Context, n keepNeed) { // keepRevision keeps c when the store holds it, and says whether a revision was kept now. func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool { + fresh, _ := p.keepRevisionOf(ctx, c) + return fresh +} + +// keepRevisionOf is keepRevision answering the fault the keeping met too. +func (p *passer) keepRevisionOf(ctx context.Context, c assigngraph.Computed) (bool, error) { g := p.graph again := func() { // the next pass hands this revision again g.mu.Lock() @@ -208,10 +251,11 @@ func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool used, _, _ := g.fillOf(ctx) rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock()) if err != nil { - p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err) again() - return false + p.keepingFailed(ctx, c, err) + return false, err } + p.keepingSucceeded() var source int64 for _, cat := range rev.Catalogs { if cat.Branch == p.w.trunkName() { @@ -222,14 +266,54 @@ func (p *passer) keepRevision(ctx context.Context, c assigngraph.Computed) bool g.kept, g.ok, g.source = rev, true, source g.mu.Unlock() if !fresh { - return false + return false, nil } p.fillingAfter(ctx, used) id := fmt.Sprintf("%s-%s-%d", meshwants.EventAssignmentRevision, rev.Kind, rev.Revision) if err := p.wantsSay(meshwants.EventAssignmentRevision, assigngraph.Event(rev), id); err != nil { p.logf("the assignment revision %d is kept but not said: %v", rev.Revision, err) } - return true + return true, nil +} + +// keepNow is the seat mesh-wants' act keep-now (design 47 H2 §813 (2); H-A-417): the keeping of the pass's newest computed +// revision, run at once, one at a time with the pass's; it answers the revision kept, or the fault the keeping met (a failed +// one is run again after keeping-retry, as every keeping is). The hand act is the tool's to write, before it calls. +func (p *passer) keepNow(ctx context.Context, in meshwants.KeepNowInput) (meshwants.KeepNowAnswer, error) { + g := p.graph + if strings.TrimSpace(in.Why) == "" { + return meshwants.KeepNowAnswer{}, errors.New("say why the keeping runs now") + } + if g == nil || g.objects == nil { + return meshwants.KeepNowAnswer{}, errors.New("the assigner keeps no assignment graph here") + } + g.mu.Lock() + c, ok := g.computed, g.hasComputed + g.mu.Unlock() + if !ok { + return meshwants.KeepNowAnswer{}, errors.New("no pass has computed an assignment graph revision yet") + } + g.keepMu.Lock() + defer g.keepMu.Unlock() + p.logf("the keeping of the assignment graph runs now by hand: %s", in.Why) + if _, err := p.keepRevisionOf(ctx, c); err != nil { + return meshwants.KeepNowAnswer{Fault: err.Error()}, nil + } + rev, _, _ := g.keptNow() + return meshwants.KeepNowAnswer{Revision: &rev}, nil +} + +// keepNowVerb is the process's handler of the seat mesh-wants' act keep-now. +func keepNowVerb(ctx context.Context, p *passer) map[string]process.SeatHandler { + return map[string]process.SeatHandler{ + meshwants.VerbKeepNow: func(_ string, raw json.RawMessage) (any, error) { + var in meshwants.KeepNowInput + if err := json.Unmarshal(raw, &in); err != nil { + return nil, fmt.Errorf("the arguments are not JSON: %w", err) + } + return p.keepNow(ctx, in) + }, + } } // graphInputs are what the pass's assignment graph is computed from: every followed branch's catalog half, the placements, each @@ -366,3 +450,63 @@ func (p *passer) graphKept() (meshwants.AssignmentRevision, int64, bool) { } return p.graph.keptNow() } + +// keepingFailed marks the keeping of c wanted again after keeping-retry (design 47 H2 §813 (2)): run then unless a newer +// revision was handed meanwhile, whose own keeping runs. The third failure in a row raises keeping-failed (scope module). +func (p *passer) keepingFailed(ctx context.Context, c assigngraph.Computed, err error) { + g := p.graph + g.mu.Lock() + g.failures++ + n, retry, after := g.failures, g.retry, g.after + g.mu.Unlock() + if retry <= 0 { + retry = defaultKeepingRetry + } + if after == nil { + after = func(d time.Duration, f func()) { time.AfterFunc(d, f) } + } + p.logf("the assignment graph revision is not kept (%d in a row), so no want is said as wanted until it is; kept again in %s: %v", n, retry, err) + if n == keepingFailedAt && p.findings.Put != nil { + if err := p.findings.Put(keepingFailedFinding(n, err).Finding(p.clock())); err != nil { + p.logf("the finding %s is not written: %v", kindKeepingFailed, err) + } + } + after(retry, func() { + g.mu.Lock() + newer := g.handed && !assigngraph.Same(g.handedRev, c.Revision) + if !newer { + g.handedRev, g.handed = c.Revision, true + } + g.mu.Unlock() + if !newer { + p.needKeeping(ctx, keepNeed{c: &c}) + } + }) +} + +// keepingSucceeded ends a run of failures: keeping-failed, raised at the third, is cleared. +func (p *passer) keepingSucceeded() { + g := p.graph + g.mu.Lock() + n := g.failures + g.failures = 0 + g.mu.Unlock() + if n >= keepingFailedAt && p.findings.Delete != nil { + o := keepingFailedFinding(n, nil) + if err := p.findings.Delete(views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)); err != nil { + p.logf("the finding %s is not deleted: %v", kindKeepingFailed, err) + } + } +} + +// keepingFailedFinding is keeping-failed of the assigner on this node, after n failures in a row (design 47 H2 §813 (2)). +func keepingFailedFinding(n int, err error) assign.Observed { + summary := fmt.Sprintf("the keeping of the assignment graph revisions failed %d times in a row", n) + if err != nil { + summary += ": " + err.Error() + } + return assign.Observed{Scope: mcc.ScopeModule, ID: assignment.Module() + "." + thisNode, Kind: kindKeepingFailed, + Severity: mcseat.FindingSeverityWarning, Summary: summary, Headline: "The assignment graph is not kept", + Explanation: summary + ". No want is said as wanted until a keeping succeeds; it is run again after keeping-retry.", + Source: "assigner." + kindKeepingFailed} +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go index 5acdb4241..d75874ac6 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept_test.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "slices" "strings" @@ -11,6 +12,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -18,6 +20,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/catalogue" + "mesh-delivery-assigner/internal/assign" "mesh-delivery-assigner/internal/assigngraph" "mesh-delivery-assigner/internal/buckets" ) @@ -273,4 +276,65 @@ func TestAPassWritesWantsWhileAKeepingIsBlocked(t *testing.T) { if !said("the pass took ") { t.Error("the pass does not say its duration") } + open() + kept.Wait() + // Design 47 H2 §813 (2): a keeping that fails runs again after keeping-retry with no pass, raising keeping-failed at the third + // failure in a row, cleared at the next that succeeds. + t.Run("a failed keeping runs again", func(t *testing.T) { + failing := &failingObjects{blockedObjects: objects, fail: 3} + var retries []time.Duration + var put []mcseat.Finding + var deleted []string + p.findings = assign.Findings{Put: func(f mcseat.Finding) error { put = append(put, f); return nil }, + Delete: func(key string) error { deleted = append(deleted, key); return nil }} + p.graph.objects, p.graph.background, p.graph.retry = failing, nil, 30*time.Second + p.graph.after = func(d time.Duration, f func()) { retries = append(retries, d); f() } + placeOn(t, p.w, "shanks", "web") // new inputs: a keeping is handed + pass() + if len(retries) != 3 || retries[0] != 30*time.Second || len(put) != 1 || put[0].Kind != kindKeepingFailed || put[0].Scope != "module" || + len(deleted) != 1 || !strings.Contains(deleted[0], kindKeepingFailed) || p.graph.failures != 0 { + t.Fatalf("retries %v, raised %+v, cleared %v, failures %d", retries, put, deleted, p.graph.failures) + } + if _, _, ok := p.graph.keptNow(); !ok || failing.fail != 0 { + t.Fatal("the revision is not kept once the store takes it") + } + // keep-now (H-A-417): the keeping by hand answers the revision kept, or the fault met; it needs a why. + if _, err := p.keepNow(context.Background(), meshwants.KeepNowInput{}); err == nil { + t.Error("a keeping by hand with no why runs") + } + if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "the store was full"}); err != nil || a.Revision == nil || a.Fault != "" { + t.Errorf("keep-now answered %+v, %v; want the revision kept", a, err) + } + failing.fail, retries = 2, nil + p.graph.after = func(d time.Duration, _ func()) { retries = append(retries, d) } + placeOn(t, p.w, "novox", "web") // a new revision, whose keeping the store refuses at the pass and again by hand + pass() + if a, err := p.keepNow(context.Background(), meshwants.KeepNowInput{Why: "again"}); err != nil || a.Revision != nil || !strings.Contains(a.Fault, "the store answers nothing") || len(retries) != 2 { + t.Errorf("keep-now on a refusing store answered %+v, %v, retries %v; want the fault and a retry", a, err, retries) + } + var asked []string + tool := keepNowTools(func(key string, body any) (json.RawMessage, error) { + raw, _ := json.Marshal(body) + asked = append(asked, key+" "+string(raw)) + return json.RawMessage(`{}`), nil + })[0] + if _, err := tool.Run(map[string]any{"why": "by hand"}); err != nil || len(asked) != 2 || !strings.HasPrefix(asked[0], "seat:mesh-hand-acts.hand-act ") || + asked[1] != `seat:mesh-wants.keep-now {"why":"by hand"}` { + t.Errorf("assigner_keep_now: %v, asked %v", err, asked) + } + }) +} + +// failingObjects refuses the next fail puts, then is objects. +type failingObjects struct { + *blockedObjects + fail int +} + +func (f *failingObjects) Put(ctx context.Context, name string, data []byte) error { + if f.fail > 0 { + f.fail-- + return errors.New("the store answers nothing") + } + return f.blockedObjects.Put(ctx, name, data) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index cc6e8168b..71c2b5d21 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -619,7 +619,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, objects := newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects) // The keeping and removing of revisions runs off the pass loop, and a keeping that changed what the pass reads wakes // one (design 47 H-A-259). - p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake, + retry, err := keepingRetry() + if err != nil { + return err + } + p.graph = &graphKeeper{objects: objects, halves: iv.h.views, background: func(f func()) { go f() }, wake: wake, retry: retry, reads: newBusObjects(js, process.Account{Node: thisNode, Module: assignment.Module()}, assigngraph.RevisionsObjects)} keep := func(ctx context.Context, ev meshcatalog.CatalogRevisionEvent) error { return p.keepCandidate(ctx, iv.call, ev) @@ -706,6 +710,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } verbs := compositionVerbs(runs, steps, record) maps.Copy(verbs, dropVerb(steps)) // the seat's act drop (§778, H-A-404) + if p.graph != nil { + maps.Copy(verbs, keepNowVerb(ctx, p)) // the keeping by hand (§813 (2), H-A-417) + } stopVerbs, err := e.AnswerSeat(meshwants.Seat, seatlist.Definition(gate.Reader, meshwants.Seat), verbs) if err != nil { logger.Printf("the verbs %s, %s and %s are not answered until the assigner starts again: %v", meshwants.VerbCompositions, meshwants.VerbCompositionsDrop, meshwants.VerbDrop, err) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 94437bcb3..309e0395e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -418,6 +418,7 @@ func serveTools() error { tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) tools = append(tools, wantDropTools(runtimeStore{}, stdio.Ask)...) tools = append(tools, asksTools(stdio.QueueWaiting, stdio.State(process.TakenState), stdio.Ask)...) // the seat's work queue (§777) + tools = append(tools, keepNowTools(stdio.Ask)...) // the keeping by hand (§813 (2)) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 1bdb3c35d..af6afbc90 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -93,7 +93,7 @@ func TestManifest(t *testing.T) { // revision, entry and difference from its revisions kept as objects in the old section, form 1, capped at 512 MiB. if len(m.Claims) != 2 || m.Claims[0].Name != meshmoduleassigner.Seat || m.Claims[0].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[0].Serves, []string{"assignments", "proposals", "propose", "withdraw", "again", "placement"}) || - m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop", "drop"}) { + m.Claims[1].Name != meshwants.Seat || m.Claims[1].At() != catalogue.ScopeMesh || !slices.Equal(m.Claims[1].Serves, []string{"upgrade", "bus", "revision", "entry", "difference", "input", "compositions", "compositions-drop", "keep-now", "drop"}) { t.Errorf("the assigner claims %+v, want mesh-module-assigner with its five verbs and mesh-wants with its five", m.Claims) } if len(m.Objects) != 1 || m.Objects[0].Name != "assignment-revisions" || m.Objects[0].MaxMiB != 512 || m.Objects[0].Form != 1 { @@ -105,7 +105,7 @@ func TestManifest(t *testing.T) { // which lists and asks again what each node's engine reads. if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_waits", "assigner_inputs", "assigner_inputs_drop", "assigner_compositions", "assigner_compositions_drop", "assignments", "seat-holders", "wants", "asks", "assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_differences", "assigner_read_copies", - "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop"}) { + "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written", "assigner_why", "assigner_want_drop", "assigner_asks", "assigner_ask_drop", "assigner_keep_now"}) { t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) assigner_ask_again (ADR 0668 §5) and assigner_why (design 47 G A-439)", m.Tools) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { @@ -172,11 +172,11 @@ func TestManifest(t *testing.T) { } file, _ := os.ReadFile("module.json") if err := json.Unmarshal(file, &raw); err != nil || raw.Config["outside-wait-minutes"].Default != float64(30) || raw.Config["outside-settle-seconds"].Default != float64(90) || - raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 14 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || + raw.Config["ask-operator-for"].Default != "take-off,remove-node,stops" || len(raw.Config) != 15 || raw.Config["tier-bound"].Default != float64(30) || raw.Config["entry-asks"].Default != float64(5) || raw.Config["composition-runs-cap-mib"].Default != float64(16) || raw.Config["composition-runs-kept"].Default != "24h" || raw.Config["composition-step-bound"].Default != "30m" || raw.Config["run-untaken-bound"].Default != "24h" || raw.Config["blame-tries"].Default != float64(20) || // design 47 H2 §775, H-A-381 (2): the taken key's bound and the kept-call pace, runtime settings - raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" || raw.Config["compose-ask-bound"].Default != "10s" { + raw.Config["ask-taken-bound"].Default != "10m" || raw.Config["kept-call-pace-first"].Default != "1s" || raw.Config["kept-call-pace-max"].Default != "30s" || raw.Config["compose-ask-bound"].Default != "10s" || raw.Config["keeping-retry"].Default != "30s" { t.Errorf("config = %+v (%v), want outside-wait-minutes of 30, outside-settle-seconds of 90 and ask-operator-for of take-off,remove-node,stops (A-425a)", raw.Config, err) } for _, f := range []struct{ env, path, key string }{ diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 699f5dfd1..3bb05e6ef 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -28,6 +28,7 @@ "input", "compositions", "compositions-drop", + "keep-now", "drop" ] } @@ -132,7 +133,8 @@ "assigner_why", "assigner_want_drop", "assigner_asks", - "assigner_ask_drop" + "assigner_ask_drop", + "assigner_keep_now" ], "state": [ { "name": "assignments", "history": 64 }, @@ -245,6 +247,13 @@ "default": "10s", "runtime": true }, + "keeping-retry": { + "type": "string", + "description": "how long after a failed keeping of the assignment graph revisions the assigner keeps them again; keeping-failed is raised at the third failure in a row, a Go duration such as 30s (design 47 H2 §813 (2)).", + "trusted": false, + "default": "30s", + "runtime": true + }, "kept-call-pace-first": { "type": "string", "description": "the first wait before a kept call met by a transport fault is made again, doubled per call up to kept-call-pace-max, a Go duration (design 47 H2 §775).", diff --git a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go index e83bbf8e2..b13b8b95d 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshwants/contract.go @@ -20,7 +20,7 @@ const Form = 2 // whose fields changed while its forms did not (design 47 H2 §780). const ( generatedForms = "2" - generatedFieldsDigest = "Ask=6cb8b82e2fbe AskTakenEvent=6d5d3da7ac39 AsksByAssignmentsProposalValue=0bb5682c7b3b AsksByBackupNodeModuleFingerprintValue=0bb5682c7b3b AsksByHoldModuleFingerprintValue=0bb5682c7b3b AsksByOutsideNodeModuleFingerprintValue=0bb5682c7b3b AsksByRecordNodeModuleDigestValue=0bb5682c7b3b AsksByRestoreNodeModuleFingerprintValue=0bb5682c7b3b AsksByRevertNodeModuleFingerprintValue=0bb5682c7b3b AssignmentEntry=7d2549782eb2 AssignmentRevision=72c8701f7314 AssignmentRevisionCapabilitiesItem=0c89b0717762 AssignmentRevisionEvent=7faff968ce44 AssignmentRevisionsByCandidateGroupHeadsValue=8553b390118d AssignmentRevisionsByEntryDigestValue=4e5ee7a23e5b AssignmentRevisionsByTrunkRevisionValue=8553b390118d BusAnswer=7538f2f5e68e CatalogOf=7a4646fdbea7 Change=28d1f3c5cd32 ChangeAfterItem=30d718c21581 CompositionReader=5a943c1f8ef1 CompositionRun=a64bc503c99a CompositionRunProgram=5a1202df3497 CompositionsAnswer=f8bca93c7f9c CompositionsByGroupHeadsValue=9428965fab8f CompositionsDropAnswer=52803700282a Difference=4011cff516b1 DifferenceAnswer=629237aa39f0 DifferenceEvent=49066d0dff6d DifferenceRefusedItem=0e9caa83ace7 DifferencesByNodeModuleValue=a22487ffadfd DropAnswer=7538f2f5e68e Edge=a71709025a08 EntryAnswer=05adebad70d6 Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 Held=15abbd6a1a33 HeldCandidate=04872f6afbfa InputAnswer=cd41ee65565d NodeEntryRef=3888ae847653 NotRunning=279b628db0ec PipelineWanted=e1e379c21e59 PipelineWantedEvent=59946b31ab7d PolicyChangedEvent=7538f2f5e68e RevisionAnswer=39002c770cbd RoomEvent=5bd4df78ed84 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 StepAccept=18efeefb6c34 StepEnded=edd08ab1a18b StepEndedEvent=207787118dac StepProgressEvent=8215e490ea8d StepRun=b975f402601a UpgradeAnswer=7538f2f5e68e Wait=951500be820c WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 WantedEvent=58cf7a9f646b WantsByHeldModuleFingerprintValue=ff14aa998a5d WantsByWantNodeModuleValue=ab7c6799fd5d WantsByWantNodeNodeValue=a737a2108d53" + generatedFieldsDigest = "Ask=6cb8b82e2fbe AskTakenEvent=6d5d3da7ac39 AsksByAssignmentsProposalValue=0bb5682c7b3b AsksByBackupNodeModuleFingerprintValue=0bb5682c7b3b AsksByHoldModuleFingerprintValue=0bb5682c7b3b AsksByOutsideNodeModuleFingerprintValue=0bb5682c7b3b AsksByRecordNodeModuleDigestValue=0bb5682c7b3b AsksByRestoreNodeModuleFingerprintValue=0bb5682c7b3b AsksByRevertNodeModuleFingerprintValue=0bb5682c7b3b AssignmentEntry=7d2549782eb2 AssignmentRevision=72c8701f7314 AssignmentRevisionCapabilitiesItem=0c89b0717762 AssignmentRevisionEvent=7faff968ce44 AssignmentRevisionsByCandidateGroupHeadsValue=8553b390118d AssignmentRevisionsByEntryDigestValue=4e5ee7a23e5b AssignmentRevisionsByTrunkRevisionValue=8553b390118d BusAnswer=7538f2f5e68e CatalogOf=7a4646fdbea7 Change=28d1f3c5cd32 ChangeAfterItem=30d718c21581 CompositionReader=5a943c1f8ef1 CompositionRun=a64bc503c99a CompositionRunProgram=5a1202df3497 CompositionsAnswer=f8bca93c7f9c CompositionsByGroupHeadsValue=9428965fab8f CompositionsDropAnswer=52803700282a Difference=4011cff516b1 DifferenceAnswer=629237aa39f0 DifferenceEvent=49066d0dff6d DifferenceRefusedItem=0e9caa83ace7 DifferencesByNodeModuleValue=a22487ffadfd DropAnswer=7538f2f5e68e Edge=a71709025a08 EntryAnswer=05adebad70d6 Fill=80f442916429 FillingEvent=5bd4df78ed84 FullEvent=5bd4df78ed84 Held=15abbd6a1a33 HeldCandidate=04872f6afbfa InputAnswer=cd41ee65565d KeepNowAnswer=9962ab9fc051 NodeEntryRef=3888ae847653 NotRunning=279b628db0ec PipelineWanted=e1e379c21e59 PipelineWantedEvent=59946b31ab7d PolicyChangedEvent=7538f2f5e68e RevisionAnswer=39002c770cbd RoomEvent=5bd4df78ed84 Shared=c981cf6aa860 SharedAskedBy=c9b30cc4d0d0 StepAccept=18efeefb6c34 StepEnded=edd08ab1a18b StepEndedEvent=207787118dac StepProgressEvent=8215e490ea8d StepRun=b975f402601a UpgradeAnswer=7538f2f5e68e Wait=951500be820c WaitEndedEvent=2bf6d38041c0 WaitHeldEvent=2bf6d38041c0 WantedEvent=58cf7a9f646b WantsByHeldModuleFingerprintValue=ff14aa998a5d WantsByWantNodeModuleValue=ab7c6799fd5d WantsByWantNodeNodeValue=a737a2108d53" ) const ( @@ -32,6 +32,7 @@ const ( VerbInput = "input" VerbCompositions = "compositions" VerbCompositionsDrop = "compositions-drop" + VerbKeepNow = "keep-now" VerbDrop = "drop" ) @@ -57,7 +58,7 @@ const ( // Verbs is every verb the seat serves, in its definition's order: a new copy each call. func Verbs() []string { - return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput, VerbCompositions, VerbCompositionsDrop, VerbDrop} + return []string{VerbUpgrade, VerbBus, VerbRevision, VerbEntry, VerbDifference, VerbInput, VerbCompositions, VerbCompositionsDrop, VerbKeepNow, VerbDrop} } // Events is every event published on the seat's subject: a new copy each call. @@ -1246,6 +1247,22 @@ type CompositionsDropAnswer struct { Run CompositionRun `json:"run"` } +// KeepNowInput is what keep-now takes. +type KeepNowInput struct { + // why the keeping runs now + Why string `json:"why"` +} + +// KeepNowAnswer is what keep-now answers. +type KeepNowAnswer struct { + // the fault the keeping met; absent when it kept the revision + Fault string `json:"fault,omitempty"` + // is the form of what this value carries, left out while it is 1 (design 47 H2 §780). + Form int64 `json:"form,omitempty"` + // the revision kept + Revision *AssignmentRevision `json:"revision,omitempty"` +} + // DropInput is what drop takes. type DropInput struct { // who drops it @@ -1557,6 +1574,15 @@ func DecodeCompositionsDropAnswer(raw []byte) (CompositionsDropAnswer, int, erro return v, formRead(raw), nil } +// DecodeKeepNowAnswer reads a KeepNowAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). +func DecodeKeepNowAnswer(raw []byte) (KeepNowAnswer, int, error) { + var v KeepNowAnswer + if err := json.Unmarshal(raw, &v); err != nil { + return v, 0, err + } + return v, formRead(raw), nil +} + // DecodeDropAnswer reads a DropAnswer as another part wrote it: a field this reader does not know is left unread, never refused; the form read is answered, 1 when absent (design 47 H2 §780). func DecodeDropAnswer(raw []byte) (DropAnswer, int, error) { var v DropAnswer diff --git a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json index e13f7058b..3b72c9c4e 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-wants/seat.json @@ -278,6 +278,36 @@ ] } }, + { + "name": "keep-now", + "description": "Keep the assignment graph's revisions now, by hand (design 47 H2 §813 (2); H-A-417): written in the hand-act log first by the module's tool assigner_keep_now, then the keeping runs at once on the pass's newest computed revision; answers the revision kept, or the fault the keeping met.", + "input": { + "type": "object", + "properties": { + "why": { + "type": "string", + "description": "why the keeping runs now" + } + }, + "required": [ + "why" + ] + }, + "answer": { + "type": "object", + "properties": { + "revision": { + "$ref": "#/defs/AssignmentRevision", + "description": "the revision kept" + }, + "fault": { + "type": "string", + "description": "the fault the keeping met; absent when it kept the revision" + } + } + }, + "class": "act" + }, { "name": "drop", "description": "End an ask on the seat's work queue for a caller that no longer waits for it (its step ended by its bound): one waiting is ended dropped when it is pulled, one taken is cancelled; both end the step dropped, naming who dropped it and why, so no holder works on it after (design 47 H2 §778; ADR 0664 §4).", diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts index b9f4ef056..dd3efa85d 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.d.ts @@ -13,6 +13,7 @@ export declare const verbs: { readonly input: "input"; readonly compositions: "compositions"; readonly compositionsDrop: "compositions-drop"; + readonly keepNow: "keep-now"; readonly drop: "drop"; }; @@ -672,6 +673,22 @@ export interface CompositionsDropAnswer { "run": CompositionRun; } +/** is what keep-now takes. */ +export interface KeepNowInput { + /** why the keeping runs now */ + "why": string; +} + +/** is what keep-now answers. */ +export interface KeepNowAnswer { + /** the fault the keeping met; absent when it kept the revision */ + "fault"?: string; + /** is the form of what this value carries, left out while it is 1 (design 47 H2 §780). */ + "form"?: number; + /** the revision kept */ + "revision"?: AssignmentRevision; +} + /** is what drop takes. */ export interface DropInput { /** who drops it */ diff --git a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js index 7c9016b74..94fb35ed8 100644 --- a/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js +++ b/modules/mesh-delivery-contracts/ts/seats/meshwants/index.js @@ -13,6 +13,7 @@ export const verbs = { input: "input", compositions: "compositions", compositionsDrop: "compositions-drop", + keepNow: "keep-now", drop: "drop", };