From 00b52e3c1ff9c16bd4e88a3e8a62714d57be9a4d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:33:26 +0100 Subject: [PATCH 01/24] BG4, BN4, BI4: the puller says ask-taken, step-progress and the end of any ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A holder says ask-taken when an ask starts, step-progress every minute while it runs (phase fetch, build or run) and step-ended for an ask outside a pipeline, each on the seat's subject of its node (SayOn); the asker is the last token of the subject the ask came on (Pulled.Subject). Builds: design 47 G §496, §602, §603, §481 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-delivery-sdk/steprun/phase.go | 7 ++ modules/mesh-delivery-sdk/steprun/pull.go | 103 +++++++++++++++++- modules/mesh-delivery-sdk/steprun/pull_bus.go | 2 +- .../steprun/pull_events_test.go | 93 ++++++++++++++++ modules/mesh-delivery-sdk/steprun/steprun.go | 12 ++ 5 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-sdk/steprun/pull_events_test.go diff --git a/modules/mesh-delivery-sdk/steprun/phase.go b/modules/mesh-delivery-sdk/steprun/phase.go index a4e878a7..72d38465 100644 --- a/modules/mesh-delivery-sdk/steprun/phase.go +++ b/modules/mesh-delivery-sdk/steprun/phase.go @@ -65,3 +65,10 @@ func (r *phases) end() sdkstep.Step { r.ended = true return r.step } + +// phase is the phase the ask is in now, "" before its first. +func (r *phases) phase() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.step.Phase +} diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index c3599866..26677b54 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -27,6 +27,9 @@ type Pulled struct { // Kind is the worker it came from, filled by the puller. Kind string Data []byte + // Subject is the subject the ask was published on: its last token is the asker the bus proved (design 47 G §481); empty + // for a source that has none, and for main's form, which names none. + Subject string // Delivered is how many times the bus has handed it over, this time included. Delivered uint64 // Ack ends it: it is never delivered again. InProgress tells the bus it is still being worked, restarting its @@ -53,6 +56,8 @@ var ( // DefaultHeartbeat is how often a running ask tells the bus it is still being worked; under the seat worker's // acknowledgement wait of 60 s (mesh-controller internal/broker HolderConsumerFor). DefaultHeartbeat = 20 * time.Second + // DefaultProgressEvery is how often a running ask says step-progress (design 47 G §496: once a minute). + DefaultProgressEvery = time.Minute ) // Puller pulls a build seat's asks and runs them. @@ -72,7 +77,12 @@ type Puller struct { // sdkstep.AgainAfter (a minute, then four). A test's own is short. Delay func(run int) time.Duration Heartbeat time.Duration - Logf func(format string, args ...any) + // Say publishes one event of the seat on this node's subject (SayOn): ask-taken when an ask starts, step-progress once + // every ProgressEvery while it runs, step-ended when an ask outside a pipeline ends (design 47 G §496, §602). Nil says + // none, a test's. ProgressEvery defaults to DefaultProgressEvery. + Say func(event string, body any, id string) error + ProgressEvery time.Duration + Logf func(format string, args ...any) // Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull // reads the node's pause key (sdkstep.PauseKey) before it pulls, and Pause and Resume write it with PausePut, so a // pause outlives a restart. @@ -665,6 +675,7 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.running[ask.ID] = RunningAsk{ID: ask.ID, Kind: ask.Kind, Module: module, Started: step.Started} p.mu.Unlock() p.logf("the %s ask %s starts on %s: %s", ask.Kind, ask.ID, p.Node, step.What) + p.say("ask-taken", askTaken{Ask: ask.ID, Asker: askerOf(msg, ask), Node: p.Node, At: now().UTC()}, "ask-taken."+ask.ID) go p.run(ctx, msg, ask, key, step) } @@ -696,6 +707,25 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key } }() ph := &phases{put: p.Put, key: key, step: step, logf: p.Logf, prefix: fmt.Sprintf("the %s ask %s on %s", ask.Kind, ask.ID, p.Node)} + if p.Say != nil { + every := p.ProgressEvery + if every <= 0 { + every = DefaultProgressEvery + } + go func() { + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-done: + return + case <-t.C: + p.say("step-progress", stepProgress{Ask: ask.ID, Asker: askerOf(msg, ask), Phase: progressPhase(ask, ph.phase()), At: now().UTC()}, + fmt.Sprintf("step-progress.%s.%d", ask.ID, now().UnixNano())) + } + } + }() + } runCtx := p.cancellable(ctx, ask.ID) result, err := protect(runCtx, func(ctx context.Context) (any, error) { return p.Run(context.WithValue(ctx, phaseKey{}, ph), ask) @@ -713,6 +743,7 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key var dec *Declining if errors.As(err, &dec) && ctx.Err() == nil { p.declineRun(msg, ask, key, step, dec) + p.sayEnded(msg, ask, sdkstep.OutcomeCouldNotRun, dec.Text) return } if ctx.Err() != nil { @@ -750,6 +781,9 @@ func (p *Puller) run(ctx context.Context, msg *Pulled, ask sdkstep.StepAsk, key if len(outcome) > 300 { outcome = outcome[:300] + "..." } + p.sayEnded(msg, ask, sdkstep.OutcomeFailed, err.Error()) + } else { + p.sayEnded(msg, ask, sdkstep.OutcomePassed, "") } p.logf("the %s ask %s ended on %s after %s, in its phase %q; %s", ask.Kind, ask.ID, p.Node, now().Sub(step.Started).Round(time.Second), step.Phase, outcome) @@ -884,3 +918,70 @@ func askRead(ask sdkstep.StepAsk) sdkstep.Ask { out, _ := sdkstep.AskAs[sdkstep.Ask](ask) return out } + +// askTaken and stepProgress are the bodies of the seat's events ask-taken and step-progress (design 47 G §602, §496). +type askTaken struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + Node string `json:"node"` + At time.Time `json:"at"` +} + +type stepProgress struct { + Ask string `json:"ask"` + Asker string `json:"asker"` + Phase string `json:"phase"` + At time.Time `json:"at"` +} + +// askerOf is the asker the bus proved: the last token of the subject the ask came on (design 47 G §481), the ask's own +// field only for a source that gives no subject and for main's form. +func askerOf(msg *Pulled, ask sdkstep.StepAsk) string { + if _, _, asker, err := sdkstep.ParseAskSubjectBy(msg.Subject); err == nil && asker != "" { + return asker + } + return ask.Asker +} + +// progressPhase is the phase step-progress says (fetch, build, run or report): fetch until the ask has entered a phase of +// its own or while it names the fetch, then the ask's kind (build for a build, run for any other). +func progressPhase(ask sdkstep.StepAsk, phase string) string { + switch { + case phase == "" || strings.HasPrefix(phase, "fetch"): + return "fetch" + case ask.Build != nil: + return "build" + } + return "run" +} + +// say publishes one event of the seat; a failure is logged and never fails the ask. +func (p *Puller) say(event string, body any, id string) { + if p.Say == nil { + return + } + if err := p.Say(event, body, id); err != nil { + p.logf("the event %s of the ask %s could not be said on %s: %v", event, id, p.Node, err) + } +} + +// sayEnded says step-ended for an ask outside a pipeline, whatever its asker (design 47 G §602): a pipeline step's end is +// its run's own (AskSteps.SayStep). +func (p *Puller) sayEnded(msg *Pulled, ask sdkstep.StepAsk, outcome, said string) { + if p.Say == nil || ask.Step != nil { + return + } + if len(said) > 300 { + said = said[:300] + "..." + } + if said == "" && outcome != sdkstep.OutcomePassed { + said = "the ask did not pass" + } + end := sdkstep.StepEnded{ID: ask.ID, Asker: askerOf(msg, ask), Ask: ask.ID, Node: p.Node, Outcome: outcome, Said: said, At: now().UTC()} + body, err := sdkstep.EncodeStepEnded(end) + if err != nil { + p.logf("the end of the ask %s cannot be said on %s: %v", ask.ID, p.Node, err) + return + } + p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID) +} diff --git a/modules/mesh-delivery-sdk/steprun/pull_bus.go b/modules/mesh-delivery-sdk/steprun/pull_bus.go index efedb295..e261a1d0 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_bus.go +++ b/modules/mesh-delivery-sdk/steprun/pull_bus.go @@ -69,7 +69,7 @@ func (s *JetStreamAsks) Next(ctx context.Context, kind string, wait time.Duratio if meta, err := msg.Metadata(); err == nil && meta != nil { delivered = meta.NumDelivered } - return &Pulled{Data: append([]byte(nil), msg.Data()...), Delivered: delivered, + return &Pulled{Data: append([]byte(nil), msg.Data()...), Subject: msg.Subject(), Delivered: delivered, Ack: msg.Ack, InProgress: msg.InProgress, Nak: msg.Nak, Term: msg.Term, NakAfter: func(delay time.Duration) error { return msg.NakWithDelay(delay) }}, nil case <-ctx.Done(): diff --git a/modules/mesh-delivery-sdk/steprun/pull_events_test.go b/modules/mesh-delivery-sdk/steprun/pull_events_test.go new file mode 100644 index 00000000..466cc951 --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/pull_events_test.go @@ -0,0 +1,93 @@ +package steprun + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" +) + +type said struct { + event, id string + body map[string]any +} + +type sayer struct { + mu sync.Mutex + got []said +} + +func (s *sayer) say(event string, body any, id string) error { + raw, _ := json.Marshal(body) + var m map[string]any + _ = json.Unmarshal(raw, &m) + s.mu.Lock() + s.got = append(s.got, said{event, id, m}) + s.mu.Unlock() + return nil +} + +func (s *sayer) of(event string) []said { + s.mu.Lock() + defer s.mu.Unlock() + var out []said + for _, g := range s.got { + if g.event == event { + out = append(out, g) + } + } + return out +} + +// Design 47 G §496, §602 (BG4, BN4, BI4): a holder says ask-taken when an ask starts, step-progress while it runs and +// step-ended when an ask outside a pipeline ends, whatever its asker. +func TestPullSaysAskTakenProgressAndEnd(t *testing.T) { + for _, c := range []struct { + name string + runErr error + outcome string + }{{"passed", nil, sdkstep.OutcomePassed}, {"failed", errors.New("the build broke"), sdkstep.OutcomeFailed}} { + t.Run(c.name, func(t *testing.T) { + src, st, sy := NewFakeAsks(), newSteps(), &sayer{} + a := testAsk(sdkstep.AskDelivery) + _ = src.PublishAsk(a) + release, entered := make(chan struct{}), make(chan struct{}) + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, + PullWait: 10 * time.Millisecond, Heartbeat: time.Hour, Say: sy.say, ProgressEvery: 5 * time.Millisecond, + Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { + Phase(ctx, "build m") + close(entered) + <-release + return nil, c.runErr + }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + <-entered + waitFor(t, "a step-progress in the build", func() bool { + pr := sy.of("step-progress") + return len(pr) > 0 && pr[len(pr)-1].body["phase"] == "build" + }) + close(release) + waitFor(t, "step-ended", func() bool { return len(sy.of("step-ended")) == 1 }) + taken := sy.of("ask-taken") + if len(taken) != 1 || taken[0].body["ask"] != a.ID || taken[0].body["node"] != "shanks" || taken[0].id != "ask-taken."+a.ID { + t.Fatalf("ask-taken = %+v, want one for %s on shanks", taken, a.ID) + } + if pr := sy.of("step-progress")[len(sy.of("step-progress"))-1].body; pr["ask"] != a.ID || pr["phase"] != "build" || pr["asker"] != a.Asker { + t.Fatalf("step-progress = %v, want the ask, its asker and the phase build", pr) + } + end := sy.of("step-ended")[0].body + if end["outcome"] != c.outcome || end["ask"] != a.ID || end["asker"] != a.Asker || end["node"] != "shanks" || end["at"] == nil { + t.Fatalf("step-ended = %v, want outcome %s of ask %s", end, c.outcome, a.ID) + } + if c.runErr != nil && end["said"] != "the build broke" { + t.Fatalf("step-ended said %v, want why", end["said"]) + } + }) + } +} diff --git a/modules/mesh-delivery-sdk/steprun/steprun.go b/modules/mesh-delivery-sdk/steprun/steprun.go index c8a44c0a..7cd29e6e 100644 --- a/modules/mesh-delivery-sdk/steprun/steprun.go +++ b/modules/mesh-delivery-sdk/steprun/steprun.go @@ -250,3 +250,15 @@ func SayStepOn(language, node string, publish func(subject string, body any, id return publish(subject, json.RawMessage(body), "step-ended."+end.ID) } } + +// SayOn is a holder's Puller.Say: an event of the language's build seat published on that seat's subject of node with +// publish (stdio.SeatPublish), `mesh.seat.node-build-.event..`. +func SayOn(language, node string, publish func(subject string, body any, id string) error) func(event string, body any, id string) error { + return func(event string, body any, id string) error { + seat, err := sdkstep.BuildSeat(language) + if err != nil { + return err + } + return publish("mesh.seat."+seat+".event."+event+"."+node, body, id) + } +} From 7033d5e80d1f13f5473b6b3cfd9c7d9b13ceddbf Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:42:00 +0100 Subject: [PATCH 02/24] BG2, BG3: the puller refuses a kind its asker may not ask and pulls the named worker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An ask's asker is the last token of its accept subject; the holder checks the kind against the callers the build seat's definition lists (Puller.Callers, CallersFromDefinitions), refuses an ask of another kind answering error naming the kind and the asker and ending it failed, takes only a build and a run on main's subject, and gives back an ask the seat list does not judge yet. A second worker per queue (AskNamedWorker, filtered to accept..*) is pulled after main's; the holder says `named worker not bound` while its membership lists none. Builds: design 47 G §543, §544, §481, A-163, A-170 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-delivery-sdk/buildstep/ask.go | 10 ++ modules/mesh-delivery-sdk/steprun/askers.go | 149 ++++++++++++++++++ .../mesh-delivery-sdk/steprun/askers_test.go | 86 ++++++++++ modules/mesh-delivery-sdk/steprun/pull.go | 12 +- modules/mesh-delivery-sdk/steprun/pull_bus.go | 78 +++++++-- .../mesh-delivery-sdk/steprun/pull_fake.go | 20 ++- .../steprun/pull_integration_test.go | 44 +++--- .../mesh-delivery-sdk/steprun/pull_seat.go | 52 +++++- .../steprun/pull_seat_test.go | 35 +++- 9 files changed, 449 insertions(+), 37 deletions(-) create mode 100644 modules/mesh-delivery-sdk/steprun/askers.go create mode 100644 modules/mesh-delivery-sdk/steprun/askers_test.go diff --git a/modules/mesh-delivery-sdk/buildstep/ask.go b/modules/mesh-delivery-sdk/buildstep/ask.go index 64439a89..e78e5c60 100644 --- a/modules/mesh-delivery-sdk/buildstep/ask.go +++ b/modules/mesh-delivery-sdk/buildstep/ask.go @@ -203,6 +203,16 @@ func AskWorker(language, kind string) (string, error) { return stream + "_" + strings.ToUpper(kind) + "_worker", nil } +// AskNamedWorker is the pull worker of a queue's named asks, `SEAT___named_worker`, filtered to +// `mesh.seat..accept..*`, beside main's worker AskWorker, filtered to main's subject alone (design 47 G §544). +func AskNamedWorker(language, kind string) (string, error) { + worker, err := AskWorker(language, kind) + if err != nil { + return "", err + } + return strings.TrimSuffix(worker, "_worker") + "_named_worker", nil +} + // AskStepKey is the key of the step of the ask askID on node. func AskStepKey(node, askID string) (string, error) { if err := CheckAskID(askID); err != nil { diff --git a/modules/mesh-delivery-sdk/steprun/askers.go b/modules/mesh-delivery-sdk/steprun/askers.go new file mode 100644 index 00000000..3023ad38 --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/askers.go @@ -0,0 +1,149 @@ +package steprun + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "sync/atomic" + + buildrun "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildrun" + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/seatdef" + "git.novox.be/novox/mesh-sdk/go/seatlist" +) + +// **A holder refuses a kind its asker may not ask** (design 47 G §543, A-163, A-170). An ask rides +// `mesh.seat..accept..`, the asker the asking seat's own name, and its kind stays in its body. The +// holder takes the asker from the subject and checks the kind against the callers the seat's definition lists for it (read +// from the seat list); on main's subject, which names no asker, it takes only a build and a run. + +// KindOf is the word the seat definition gives an ask's kind: run for a run or a compose, else the ask's own kind. +func KindOf(ask sdkstep.StepAsk) string { + if ask.Run != nil || ask.Compose != nil { + return "run" + } + return ask.Kind +} + +// CallersFromDefinitions is a Puller.Callers: the seats the definition of language's build seat lists as callers of an +// accepted kind, from the seat list's definitions (seatlist.Reader.Definitions). found is false when the definition or the +// kind is not held yet. A caller entry is `seat:`. +func CallersFromDefinitions(language string, definitions func() map[string]seatdef.Definition) func(kind string) (callers []string, found bool) { + return func(kind string) ([]string, bool) { + seat, err := sdkstep.BuildSeat(language) + if err != nil { + return nil, false + } + def, ok := definitions()[seat] + if !ok { + return nil, false + } + var out []string + found := false + for _, m := range def.Accepts { + if m.Name != kind { + continue + } + found = true + if m.Callers != nil { + for _, e := range m.Callers.List { + if name, ok := strings.CutPrefix(e, "seat:"); ok { + out = append(out, name) + } + } + } + } + return out, found + } +} + +// askerRefusal is why an ask must not be taken from its subject's asker: "" when it may be, and judged false when the seat +// list does not say yet (the ask is given back, not refused). +func (p *Puller) askerRefusal(msg *Pulled, ask sdkstep.StepAsk) (why string, judged bool) { + if msg.Subject == "" { + return "", true // a source that gives no subject (a test's) + } + _, _, asker, err := sdkstep.ParseAskSubjectBy(msg.Subject) + if err != nil { + return fmt.Sprintf("the ask came on %s, which is no accept subject of this seat", msg.Subject), true + } + kind := KindOf(ask) + if asker == "" { + if (ask.Build != nil && ask.Kind == sdkstep.AskDelivery) || kind == "run" { + return "", true + } + return fmt.Sprintf("the kind %s is not taken on main's subject, which takes only a build and a run (asker: none named)", kind), true + } + if p.Callers == nil { + return "", true + } + callers, found := p.Callers(kind) + if !found { + return "", false + } + if !slices.Contains(callers, asker) { + return fmt.Sprintf("the kind %s is not one %s may ask: its callers are %s", kind, asker, strings.Join(callers, ", ")), true + } + return "", true +} + +// refuseAsker ends an ask whose asker may not ask its kind: its step key says failed with the reason, naming the kind and the +// asker, step-ended says failed, and the message is dropped. +func (p *Puller) refuseAsker(msg *Pulled, ask sdkstep.StepAsk, why string) { + key, err := sdkstep.AskStepKey(p.Node, ask.ID) + if err == nil { + at := now().UTC() + step := sdkstep.Step{ID: ask.ID, Tool: ask.Kind + " ask", Node: p.Node, State: sdkstep.StepFailed, Started: at, Ended: at, + Reason: "error: " + why, Owner: buildrun.Owner, Run: int(msg.Delivered)} + err = writeEnd(p.Put, key, step) + } + if err != nil { + p.logf("the ask %s of the wrong asker cannot be answered on %s, so it is given back: %v", ask.ID, p.Node, err) + _ = msg.Nak() + return + } + p.logf("the ask %s is refused on %s: %s", ask.ID, p.Node, why) + if ask.Step == nil { + p.sayEnded(msg, ask, sdkstep.OutcomeFailed, why) + } else if end, err := sdkstep.EndOf(ask, p.Node, sdkstep.OutcomeFailed); err == nil { + // A pipeline step's end names its pipeline, step, commit and fingerprint (its own form, §602). + end.Said, end.At, end.Asker = why, now().UTC(), askerOf(msg, ask) + if body, err := sdkstep.EncodeStepEnded(end); err == nil { + p.say("step-ended", json.RawMessage(body), "step-ended."+ask.ID) + } + } + _ = msg.Term() +} + +// giveBack hands an ask back to be judged again a little later, once the seat list says more. +func (p *Puller) giveBack(msg *Pulled, ask sdkstep.StepAsk) { + p.logf("the ask %s is given back on %s: the seat list holds no definition of its kind yet", ask.ID, p.Node) + if msg.NakAfter != nil && msg.NakAfter(DefaultPullWait) == nil { + return + } + _ = msg.Nak() +} + +// SeatListCallers is a Puller.Callers that follows the seat list: it starts the reader once the runtime serves +// (seatlist.Start through the runtime's own way in) and answers from its definitions, found false until it has read them. +// A reader that cannot start is said and the asks are given back until it can (design 47 G §543, A-163). +func SeatListCallers(language string, logf func(format string, args ...any)) func(kind string) ([]string, bool) { + var reader atomic.Pointer[seatlist.Reader] + go func() { + <-stdio.Served() + r, err := seatlist.Start(seatlist.Config{Bus: seatlist.StdioBus(), Bootstrap: seatlist.Bootstrapped()}) + if err != nil && logf != nil { + logf("following the seat list: %v", err) + } + reader.Store(r) + }() + return func(kind string) ([]string, bool) { + r := reader.Load() + if r == nil { + return nil, false + } + return CallersFromDefinitions(language, r.Definitions)(kind) + } +} diff --git a/modules/mesh-delivery-sdk/steprun/askers_test.go b/modules/mesh-delivery-sdk/steprun/askers_test.go new file mode 100644 index 00000000..aa5ace6e --- /dev/null +++ b/modules/mesh-delivery-sdk/steprun/askers_test.go @@ -0,0 +1,86 @@ +package steprun + +import ( + "context" + "strings" + "testing" + "time" + + sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +func callersFor(table map[string][]string) func(string) ([]string, bool) { + return func(kind string) ([]string, bool) { c, ok := table[kind]; return c, ok } +} + +// Design 47 G §543 (BG2, BG3, A-163, A-170): an ask whose kind its asker, the last token of its subject, is no caller of is +// answered error naming the kind and the asker and ended failed; a step on main's subject is refused as well; one the +// seat list does not judge yet is given back. +func TestPullRefusesAKindItsAskerMayNotAsk(t *testing.T) { + step := testAsk(sdkstep.AskStep) + step.Check = nil + step.Step = &sdkstep.StepRun{Pipeline: "p", Step: sdkstep.StepUnit, Module: "m", Artifact: "m:a", Commit: "0123456789abcdef0123456789abcdef01234567", + Fingerprint: "fp", Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: "0123456789abcdef0123456789abcdef01234567"}} + build := testAsk(sdkstep.AskDelivery) + table := map[string][]string{"step": {"mesh-pipeline-runner"}, "delivery": {"mesh-builds"}} + for _, c := range []struct { + name, subject string + ask sdkstep.StepAsk + callers func(string) ([]string, bool) + want string // "" runs it, "refused" refuses it, "back" gives it back + }{ + {"the indexer's step", "mesh.seat.node-build-go.accept.check.mesh-catalog", step, callersFor(table), "refused"}, + {"the runner's step", "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner", step, callersFor(table), ""}, + {"a step on main's subject", "mesh.seat.node-build-go.accept.check", step, callersFor(table), "refused"}, + {"a build on main's subject", "mesh.seat.node-build-go.accept.delivery", build, callersFor(table), ""}, + {"a kind the seat list does not hold", "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner", step, callersFor(nil), "back"}, + } { + t.Run(c.name, func(t *testing.T) { + src, st, sy := NewFakeAsks(), newSteps(), &sayer{} + raw, _ := sdkstep.EncodeStepAsk(c.ask) + src.PublishWithSubject(c.ask.Kind, raw, c.subject) + ran := make(chan struct{}, 1) + p := &Puller{Source: src, Language: sdkstep.AskGo, Node: "shanks", Put: st.put, Capacity: func() int { return 1 }, Callers: c.callers, + PullWait: 5 * time.Millisecond, Heartbeat: time.Hour, Say: sy.say, ProgressEvery: time.Hour, + Run: func(context.Context, sdkstep.StepAsk) (any, error) { ran <- struct{}{}; return nil, nil }} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go p.Pull(ctx) + switch c.want { + case "": + <-ran + case "refused": + waitFor(t, "the ask ended", func() bool { _, termed, _ := src.Fates(); return len(termed) == 1 }) + key, _ := sdkstep.AskStepKey("shanks", c.ask.ID) + s, _ := st.get(key) + asker := c.subject[strings.LastIndex(c.subject, ".")+1:] + if s.State != sdkstep.StepFailed || !strings.Contains(s.Reason, KindOf(c.ask)) || (strings.Contains(c.subject, "mesh-catalog") && !strings.Contains(s.Reason, asker)) { + t.Fatalf("the step = %+v, want failed naming the kind %s and the asker", s, KindOf(c.ask)) + } + if end := sy.of("step-ended"); len(end) != 1 || end[0].body["outcome"] != sdkstep.OutcomeFailed { + t.Fatalf("step-ended = %+v, want failed", end) + } + select { + case <-ran: + t.Fatal("a refused ask ran") + default: + } + case "back": + waitFor(t, "the ask given back", func() bool { src.mu.Lock(); defer src.mu.Unlock(); return len(src.Delayed) > 0 }) + } + }) + } +} + +func TestCallersFromDefinitionsReadsTheSeatsCallers(t *testing.T) { + def := seatdef.Definition{Seat: "node-build-go", Accepts: []seatdef.Message{ + {Name: "step", Callers: &seatdef.Callers{List: []string{"seat:mesh-pipeline-runner"}}}}} + callers := CallersFromDefinitions(sdkstep.AskGo, func() map[string]seatdef.Definition { return map[string]seatdef.Definition{"node-build-go": def} }) + if got, ok := callers("step"); !ok || len(got) != 1 || got[0] != "mesh-pipeline-runner" { + t.Fatalf("callers of step = %v, %v", got, ok) + } + if _, ok := callers("run"); ok { + t.Fatal("a kind the definition does not list is found") + } +} diff --git a/modules/mesh-delivery-sdk/steprun/pull.go b/modules/mesh-delivery-sdk/steprun/pull.go index 26677b54..77a772d6 100644 --- a/modules/mesh-delivery-sdk/steprun/pull.go +++ b/modules/mesh-delivery-sdk/steprun/pull.go @@ -80,7 +80,10 @@ type Puller struct { // Say publishes one event of the seat on this node's subject (SayOn): ask-taken when an ask starts, step-progress once // every ProgressEvery while it runs, step-ended when an ask outside a pipeline ends (design 47 G §496, §602). Nil says // none, a test's. ProgressEvery defaults to DefaultProgressEvery. - Say func(event string, body any, id string) error + Say func(event string, body any, id string) error + // Callers says which seats the build seat's definition lists as callers of an accepted kind (CallersFromDefinitions); + // found is false while the seat list holds no such kind. Nil checks no named asker (design 47 G §543, A-163). + Callers func(kind string) (callers []string, found bool) ProgressEvery time.Duration Logf func(format string, args ...any) // Load reads one key of the state the node's pause is kept in (nil and no error when it is absent). When set, Pull @@ -640,6 +643,13 @@ func (p *Puller) take(ctx context.Context, msg *Pulled) { p.refuseUnread(msg, err) return } + if why, judged := p.askerRefusal(msg, ask); !judged { + p.giveBack(msg, ask) + return + } else if why != "" { + p.refuseAsker(msg, ask, why) + return + } if d := p.droppedAsk(ask.ID); d != nil { p.refuseDropped(msg, ask, d) return diff --git a/modules/mesh-delivery-sdk/steprun/pull_bus.go b/modules/mesh-delivery-sdk/steprun/pull_bus.go index e261a1d0..5669ec4b 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_bus.go +++ b/modules/mesh-delivery-sdk/steprun/pull_bus.go @@ -16,14 +16,19 @@ type JetStreamAsks struct { JS jetstream.JetStream Language string - mu sync.Mutex - cons map[string]jetstream.Consumer + mu sync.Mutex + cons map[string]jetstream.Consumer + missing map[string]bool } -func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Consumer, error) { +func (s *JetStreamAsks) consumer(ctx context.Context, kind string, named bool) (jetstream.Consumer, error) { s.mu.Lock() defer s.mu.Unlock() - if c, ok := s.cons[kind]; ok { + cacheKey := kind + if named { + cacheKey = kind + "/named" + } + if c, ok := s.cons[cacheKey]; ok { return c, nil } stream, err := sdkstep.AskStream(s.Language) @@ -31,6 +36,9 @@ func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Co return nil, err } worker, err := sdkstep.AskWorker(s.Language, kind) + if named { + worker, err = sdkstep.AskNamedWorker(s.Language, kind) + } if err != nil { return nil, err } @@ -41,17 +49,60 @@ func (s *JetStreamAsks) consumer(ctx context.Context, kind string) (jetstream.Co if s.cons == nil { s.cons = map[string]jetstream.Consumer{} } - s.cons[kind] = c + s.cons[cacheKey] = c return c, nil } -// Next is AskSource's: a fetch of one with no wait when wait is 0, else one that waits at most wait. +// Next is AskSource's: main's worker first, then the queue's named worker once it is bound (design 47 G §544), with no +// wait when wait is 0, else waiting at most wait between the two. A named worker not bound is only main's alone. func (s *JetStreamAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) { - c, err := s.consumer(ctx, kind) - if err != nil { - return nil, err + var half time.Duration + if wait > 0 { + half = wait / 2 } + for i, step := range []struct { + named bool + wait time.Duration + }{{false, 0}, {true, 0}, {false, half}, {true, wait - half}} { + if step.named && s.unbound(kind) { + continue + } + if i >= 2 && wait <= 0 { + break // the two waiting pulls are for a wait + } + c, err := s.consumer(ctx, kind, step.named) + if err != nil { + if step.named { + s.markUnbound(kind) + continue + } + return nil, err + } + if msg, err := s.fetch(ctx, c, step.wait); err != nil || msg != nil { + return msg, err + } + } + return nil, nil +} + +func (s *JetStreamAsks) unbound(kind string) bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.missing[kind] +} + +func (s *JetStreamAsks) markUnbound(kind string) { + s.mu.Lock() + if s.missing == nil { + s.missing = map[string]bool{} + } + s.missing[kind] = true + s.mu.Unlock() +} + +func (s *JetStreamAsks) fetch(ctx context.Context, c jetstream.Consumer, wait time.Duration) (*Pulled, error) { var batch jetstream.MessageBatch + var err error if wait <= 0 { batch, err = c.FetchNoWait(1) } else { @@ -93,11 +144,16 @@ func DeclareAskWorkers(ctx context.Context, js jetstream.JetStream, language str for _, kind := range sdkstep.AskKinds { worker, _ := sdkstep.AskWorker(language, kind) subject, _ := sdkstep.AskSubject(language, kind) - // Both forms of the accept subject: main's and the asker's, `.` (design 47 G §600). - if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject, subject + ".*"}, + // Two workers whose filters do not overlap: main's subject and the asker's, `.` (design 47 G §544). + if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: worker, FilterSubjects: []string{subject}, AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil { return fmt.Errorf("the worker %s: %w", worker, err) } + named, _ := sdkstep.AskNamedWorker(language, kind) + if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: named, FilterSubjects: []string{subject + ".*"}, + AckPolicy: jetstream.AckExplicitPolicy, AckWait: ackWait, MaxDeliver: 5}); err != nil { + return fmt.Errorf("the worker %s: %w", named, err) + } } return nil } diff --git a/modules/mesh-delivery-sdk/steprun/pull_fake.go b/modules/mesh-delivery-sdk/steprun/pull_fake.go index 574605dc..05e648e3 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_fake.go +++ b/modules/mesh-delivery-sdk/steprun/pull_fake.go @@ -13,7 +13,9 @@ import ( type FakeAsks struct { mu sync.Mutex waiting map[string][][]byte - added chan struct{} + // subjects are the subjects asks were published with, by their data. + subjects map[string]string + added chan struct{} // Acked, Termed and Naked are the asks' ids or data by fate, InProgress how many heartbeats were given. Acked, Termed, Naked []string // Delayed are the asks handed back to be given again after a delay (NakAfter), and Delays that delay of each. @@ -40,6 +42,17 @@ func (f *FakeAsks) Publish(kind string, data []byte) { } } +// PublishWithSubject puts an encoded ask on kind's worker as the bus would hand it over from subject (design 47 G §481). +func (f *FakeAsks) PublishWithSubject(kind string, data []byte, subject string) { + f.mu.Lock() + if f.subjects == nil { + f.subjects = map[string]string{} + } + f.subjects[string(data)] = subject + f.mu.Unlock() + f.Publish(kind, data) +} + // PublishAsk encodes a and puts it on its kind's worker. func (f *FakeAsks) PublishAsk(a sdkstep.StepAsk) error { raw, err := sdkstep.EncodeStepAsk(a) @@ -107,7 +120,10 @@ func (f *FakeAsks) pulled(data []byte) *Pulled { return nil } } + f.mu.Lock() + subject := f.subjects[string(data)] delivered := f.Delivered + f.mu.Unlock() if delivered == 0 { delivered = 1 } @@ -117,6 +133,6 @@ func (f *FakeAsks) pulled(data []byte) *Pulled { f.mu.Unlock() return nil } - return &Pulled{Data: data, Delivered: delivered, NakAfter: nakAfter, Ack: fate(&f.Acked), Nak: fate(&f.Naked), Term: fate(&f.Termed), + return &Pulled{Data: data, Subject: subject, Delivered: delivered, NakAfter: nakAfter, Ack: fate(&f.Acked), Nak: fate(&f.Naked), Term: fate(&f.Termed), InProgress: func() error { f.mu.Lock(); f.InProgress++; f.mu.Unlock(); return nil }} } diff --git a/modules/mesh-delivery-sdk/steprun/pull_integration_test.go b/modules/mesh-delivery-sdk/steprun/pull_integration_test.go index 42575013..07336764 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_integration_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_integration_test.go @@ -40,7 +40,11 @@ func publishAsk(t *testing.T, js jetstream.JetStream, a sdkstep.StepAsk) { if err != nil { t.Fatal(err) } + // A build rides main's subject; any other kind a named asker's (design 47 G §543: main takes only a build and a run). subject, _ := sdkstep.AskSubject(a.Language, a.Kind) + if a.Kind != sdkstep.AskDelivery { + subject, _ = sdkstep.AskSubjectBy(a.Language, a.Kind, "mesh-delivery-checker") + } if _, err := js.Publish(context.Background(), subject, raw); err != nil { t.Fatal(err) } @@ -197,31 +201,35 @@ func TestAnAskThatMetTheNetworkIsGivenAgainAndADroppedOneIsEnded(t *testing.T) { func contains(s, sub string) bool { return strings.Contains(s, sub) } -// Design 47 G §600: the seat's worker takes an ask put on either form of the accept subject, main's and the asker's. -func TestTheWorkerTakesBothFormsOfTheAcceptSubject(t *testing.T) { +// Design 47 G §544: main's worker takes an ask put on main's subject and the named worker one put on the asker's, the two +// filters not overlapping. +func TestTheTwoWorkersTakeTheTwoFormsOfTheAcceptSubject(t *testing.T) { js := askBus(t, 30*time.Second) stream, _ := sdkstep.AskStream(sdkstep.AskGo) worker, _ := sdkstep.AskWorker(sdkstep.AskGo, sdkstep.AskDelivery) + named, _ := sdkstep.AskNamedWorker(sdkstep.AskGo, sdkstep.AskDelivery) main, _ := sdkstep.AskSubject(sdkstep.AskGo, sdkstep.AskDelivery) - for _, subject := range []string{main, main + ".builder"} { + for _, subject := range []string{main, main + ".mesh-builds"} { if _, err := js.Publish(context.Background(), subject, []byte("{}")); err != nil { t.Fatal(err) } } - c, err := js.Consumer(context.Background(), stream, worker) - if err != nil { - t.Fatal(err) - } - batch, err := c.Fetch(2, jetstream.FetchMaxWait(3*time.Second)) - if err != nil { - t.Fatal(err) - } - var got []string - for m := range batch.Messages() { - got = append(got, m.Subject()) - _ = m.Ack() - } - if len(got) != 2 { - t.Errorf("the worker took %v, want both forms of %s", got, main) + for name, want := range map[string]string{worker: main, named: main + ".mesh-builds"} { + c, err := js.Consumer(context.Background(), stream, name) + if err != nil { + t.Fatal(err) + } + batch, err := c.Fetch(2, jetstream.FetchMaxWait(time.Second)) + if err != nil { + t.Fatal(err) + } + var got []string + for m := range batch.Messages() { + got = append(got, m.Subject()) + _ = m.Ack() + } + if len(got) != 1 || got[0] != want { + t.Errorf("%s took %v, want only %s", name, got, want) + } } } diff --git a/modules/mesh-delivery-sdk/steprun/pull_seat.go b/modules/mesh-delivery-sdk/steprun/pull_seat.go index 91e5c99b..9958e5a5 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_seat.go +++ b/modules/mesh-delivery-sdk/steprun/pull_seat.go @@ -2,6 +2,7 @@ package steprun import ( "context" + "sync" "time" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" @@ -18,10 +19,15 @@ type SeatAsks struct { Settle func(id, fate string) error // SettleAfter is stdio.SeatSettleAfter when nil (ADR 0664 §1). SettleAfter func(id string, delay time.Duration) error + // Logf says `named worker not bound` once for each queue whose named worker the membership does not list yet. + Logf func(format string, args ...any) + + mu sync.Mutex + noted map[string]bool } // Next is AskSource's. -func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) { +func (s *SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*Pulled, error) { pull, settle := s.Pull, s.Settle if pull == nil { pull = stdio.SeatPull @@ -36,7 +42,33 @@ func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*P if err := ctx.Err(); err != nil { return nil, err } - p, err := pull(worker, wait) + named, err := sdkstep.AskNamedWorker(s.Language, kind) + if err != nil { + return nil, err + } + // Main's worker first, then the named one (design 47 G §544): a holder of G pulls main's alone until its membership + // names the named worker, and says so meanwhile. A wait is shared by the two. + half := wait / 2 + p, err := pull(worker, min(wait, 0)) + if err == nil && p == nil { + var nerr error + if p, nerr = pull(named, 0); nerr != nil { + s.noteUnbound(kind, nerr) + p = nil + } else { + s.mu.Lock() + delete(s.noted, kind) + s.mu.Unlock() + } + } + if err == nil && p == nil && wait > 0 { + if p, err = pull(worker, half); err == nil && p == nil { + if p, err = pull(named, wait-half); err != nil { + s.noteUnbound(kind, err) + p, err = nil, nil + } + } + } if err != nil || p == nil { return nil, err } @@ -45,7 +77,21 @@ func (s SeatAsks) Next(ctx context.Context, kind string, wait time.Duration) (*P after = stdio.SeatSettleAfter } fate := func(f string) func() error { return func() error { return settle(p.ID, f) } } - return &Pulled{Data: p.Body, Delivered: p.Delivered, Ack: fate(stdio.SettleAck), InProgress: fate(stdio.SettleProgress), + return &Pulled{Data: p.Body, Subject: p.Subject, Delivered: p.Delivered, Ack: fate(stdio.SettleAck), InProgress: fate(stdio.SettleProgress), Nak: fate(stdio.SettleNak), Term: fate(stdio.SettleTerm), NakAfter: func(delay time.Duration) error { return after(p.ID, delay) }}, nil } + +// noteUnbound says `named worker not bound` for a queue, once until it is bound (design 47 G §544). +func (s *SeatAsks) noteUnbound(kind string, err error) { + s.mu.Lock() + if s.noted == nil { + s.noted = map[string]bool{} + } + said := s.noted[kind] + s.noted[kind] = true + s.mu.Unlock() + if !said && s.Logf != nil { + s.Logf("named worker not bound: the %s queue's named asks wait until the membership lists %s: %v", kind, "its named worker", err) + } +} diff --git a/modules/mesh-delivery-sdk/steprun/pull_seat_test.go b/modules/mesh-delivery-sdk/steprun/pull_seat_test.go index 2a4d64e6..0750b807 100644 --- a/modules/mesh-delivery-sdk/steprun/pull_seat_test.go +++ b/modules/mesh-delivery-sdk/steprun/pull_seat_test.go @@ -18,7 +18,7 @@ import ( func TestSeatAsksPullsTheKindsWorkerAndSettlesByID(t *testing.T) { var pulled []string var settled []string - src := SeatAsks{Language: sdkstep.AskGo, + src := &SeatAsks{Language: sdkstep.AskGo, Pull: func(worker string, wait time.Duration) (*stdio.Pulled, error) { pulled = append(pulled, worker) if worker != "SEAT_NODE_BUILD_GO_CHECK_worker" { @@ -36,7 +36,7 @@ func TestSeatAsksPullsTheKindsWorkerAndSettlesByID(t *testing.T) { } _ = p.InProgress() _ = p.Ack() - if len(pulled) != 2 || pulled[0] != "SEAT_NODE_BUILD_GO_DELIVERY_worker" { + if len(pulled) != 3 || pulled[0] != "SEAT_NODE_BUILD_GO_DELIVERY_worker" || pulled[1] != "SEAT_NODE_BUILD_GO_DELIVERY_named_worker" { t.Errorf("pulled %v", pulled) } if len(settled) != 2 || settled[0] != "p9 progress" || settled[1] != "p9 ack" { @@ -146,3 +146,34 @@ func TestADroppedAskIsRefusedAtPullAndARunningOneIsEnded(t *testing.T) { } func ptr[T any](v T) *T { return &v } + +// Design 47 G §544 (BG3): a holder pulls main's worker, then the queue's named worker; while its membership does not list the +// named worker it takes main's alone and says `named worker not bound` once. +func TestSeatAsksPullsTheNamedWorkerAndSaysWhenItIsNotBound(t *testing.T) { + var said []string + bound := false + src := &SeatAsks{Language: sdkstep.AskGo, Logf: func(f string, a ...any) { said = append(said, fmt.Sprintf(f, a...)) }, + Pull: func(worker string, wait time.Duration) (*stdio.Pulled, error) { + if worker != "SEAT_NODE_BUILD_GO_CHECK_named_worker" { + return nil, nil + } + if !bound { + return nil, fmt.Errorf("the module pulls no work from %s", worker) + } + return &stdio.Pulled{ID: "n1", Worker: worker, Subject: "mesh.seat.node-build-go.accept.check.mesh-pipeline-runner", Body: []byte(`{"v":1}`), Delivered: 1}, nil + }, + Settle: func(string, string) error { return nil }} + for range 2 { + if p, err := src.Next(context.Background(), sdkstep.AskCheck, 0); err != nil || p != nil { + t.Fatalf("unbound: %+v, %v", p, err) + } + } + if len(said) != 1 || !strings.Contains(said[0], "named worker not bound") { + t.Fatalf("said %q, want `named worker not bound` once", said) + } + bound = true + p, err := src.Next(context.Background(), sdkstep.AskCheck, 0) + if err != nil || p == nil || !strings.HasSuffix(p.Subject, ".mesh-pipeline-runner") { + t.Fatalf("bound: %+v, %v", p, err) + } +} From b90e767529cd3b340a762b2f1c92b366f32c6d5a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:45:50 +0100 Subject: [PATCH 03/24] IX17, IX18: the indexer keeps the records pinning each catalog graph revision and removes a revision only on an event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State pins (4 MiB; one key per revision ., made, pins-stated) applied from a durable consumer of mesh-delivery's pinned, freed, heads-replaced and pins-stated, acknowledged after the apply; a revision is removed when no record pins it and it is not current, only while the consumer has nothing pending and after a pins-stated later than made; finding pin-missing; tools indexer_pins, indexer_unpin, indexer_consumer (the state consumers, written by mesh-sdk's process.ConsumerLedger). Builds: design 47 G §153, §178, §189, §215, §216, §579, A-169, A-171, A-176 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer-tools/main.go | 6 + .../mesh-delivery-indexer/integration_test.go | 47 +- .../cmd/mesh-delivery-indexer/main.go | 24 +- .../cmd/mesh-delivery-indexer/pins.go | 65 ++ .../cmd/mesh-delivery-indexer/takein.go | 4 + .../internal/index/candidates.go | 14 +- .../internal/index/catalog_revisions.go | 6 + .../internal/index/pending.go | 2 +- .../internal/index/pending_test.go | 2 +- .../internal/index/pins.go | 576 ++++++++++++++++++ .../internal/index/pins_test.go | 121 ++++ .../internal/index/takein.go | 7 +- .../mesh-delivery-indexer/manifest_test.go | 9 +- modules/mesh-delivery-indexer/module.json | 13 +- 14 files changed, 878 insertions(+), 18 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go create mode 100644 modules/mesh-delivery-indexer/internal/index/pins.go create mode 100644 modules/mesh-delivery-indexer/internal/index/pins_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index e1736091..d57fa4a3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -18,6 +18,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busstate" + "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-indexer/internal/index" ) @@ -299,6 +300,11 @@ func main() { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.NamedDropTool, Verb: "named-drop"}) return err })...) + tools = append(tools, index.PinsTools(keptRead{stdio.State(index.PinsState)}, revisions, asks, time.Now, func(what, why string) error { + _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.UnpinTool, Verb: "unpin"}) + return err + })...) + tools = append(tools, index.ConsumerTools(keptRead{stdio.State(process.ConsumersState)})...) tools = append(tools, index.RegistrationTools(keptRead{stdio.State(index.ProposalsState)}, asks, time.Now, func(cause, verb, what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause, Verb: verb}) return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index c44f37c8..4aa1dec0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -608,6 +608,11 @@ func startIndexerRun(t *testing.T, put func(context.Context, buildLog), before . t.Fatal(err) } } + for _, bucket := range []string{index.PinsBucket, index.Module + "_" + process.ConsumersState} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: bucket, History: 1}); err != nil { + t.Fatal(err) + } + } if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.NamedBucket, History: 1}); err != nil { t.Fatal(err) } @@ -1406,7 +1411,7 @@ func serveAssignerAssignments(t *testing.T, ctx context.Context, js jetstream.Je t.Fatal(err) } if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy, - FilterSubjects: []string{"mesh.mod." + assignerHolderModule + ".event.views.>"}}); err != nil { + FilterSubjects: []string{"mesh.mod." + assignerHolderModule + ".event.views.>", "mesh.seat.mesh-delivery.event.>"}}); err != nil { t.Fatal(err) } em := views.Emitter{Owner: assignerHolderModule, Subject: process.Account{Node: "n1", Module: assignerHolderModule}.EventSubject, @@ -1491,6 +1496,46 @@ func TestIndexerKeepsWhatAWantAndARunningBuildNameAndLetsAGoneOneLeave(t *testin } } +// TestIndexerAppliesThePinsEventsThroughItsDurableConsumer (design 47 G §153, §216, §579, A-169, A-171, A-176): a pinned of a revision the +// indexer does not hold and a pins-stated, said on the seat mesh-delivery, are applied to the state pins through its durable consumer (the +// key `.` marked missing, the record listed, `made` and `pins-stated` kept), and the consumer's facts reach the state +// consumers for indexer_consumer. +func TestIndexerAppliesThePinsEventsThroughItsDurableConsumer(t *testing.T) { + js, ctx, _, _ := startIndexer(t, func(context.Context, buildLog) {}) + say := func(subject string, body any) { + raw, _ := json.Marshal(body) + if _, err := js.Publish(ctx, subject, raw); err != nil { + t.Fatal(err) + } + } + say(meshdelivery.SubjectPinned, meshdelivery.Pins{Record: "r1", Milestone: "47/G", Catalog: []meshdelivery.PinsCatalogItem{{Branch: "main", Revision: 9}}}) + say(meshdelivery.SubjectPinsStated, meshdelivery.PinsStatedEvent{At: time.Now().UTC().Add(time.Minute)}) + pins, err := js.KeyValue(ctx, index.PinsBucket) + if err != nil { + t.Fatal(err) + } + consumers, err := js.KeyValue(ctx, index.Module+"_"+process.ConsumersState) + if err != nil { + t.Fatal(err) + } + var v index.PinValue + var f process.ConsumerFacts + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + e, err := pins.Get(ctx, "main.9") + c, cerr := consumers.Get(ctx, "n1_mesh-delivery-indexer") + if _, serr := pins.Get(ctx, index.PinsStatedKey); err == nil && cerr == nil && serr == nil && + json.Unmarshal(e.Value(), &v) == nil && json.Unmarshal(c.Value(), &f) == nil && f.LastEvent == meshdelivery.SubjectPinsStated { + break + } + } + if !slices.Equal(v.Records, []string{"r1"}) || !v.Missing { + t.Errorf("pins main.9 holds %+v within 20 s, want record r1 and missing", v) + } + if f.Stream != "EVENTS" || f.Acknowledged == 0 || f.LastEvent != meshdelivery.SubjectPinsStated { + t.Errorf("consumers holds %+v, want the pins-stated applied on EVENTS", f) + } +} + // TestIndexerTakesInAWatchedBranchOtherThanTheFirstIntoItsOwnRevision: the seat's verb source lists a second watched branch, // beta, with the merge of mesh-catalog it names as its head (design 47 G §13, §14, §291, A-93); the indexer started with the // branch listed reads source.beta, takes the head's merged message in as a pure computation, and keeps branch.beta.1, which diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 1f4e3b58..8136c950 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -439,6 +439,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro var fillFollow atomic.Pointer[versionsFill] // Kept while named (design 47 G §675): the want and the running build that name a fingerprint, heard below. named := index.NewNamed(awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.NamedBucket, "no entry is kept for a want or a running build that names it", logger.Printf)}, logger.Printf) + pinsBucket := process.AwaitBucket(ctx, js, conn, index.PinsBucket, "no record's pin on a catalog graph revision is kept, so no revision is removed", logger.Printf) + pinsStore := awaitedStore{b: pinsBucket} + consumers := process.AwaitBucket(ctx, js, conn, index.Module+"_"+process.ConsumersState, "indexer_consumer reads no consumer", logger.Printf) ix := &index.Indexer{NamedSets: named, Findings: writer, OldSections: awaitedStore{b: oldSections}, Unjudged: awaitedStore{b: unjudged}, Requirements: awaitedStore{b: requirements}, Store: kvStore{candidates}, Modules: modules, Manifests: kvStore{manifests}, Taken: awaitedStore{b: taken}, Builds: known, Verdicts: awaitedStore{b: verdicts}, SayRefused: sayRefused(conn, cred, logger.Printf), SayAccepted: sayAccepted(conn, cred), Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf, @@ -451,8 +454,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro SayCatalogRevision: afterRevision(refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow), &fillFollow), SayFill: sayFill(conn, cred), // The catalog's released versions (design 47 G §349): the state versions, bound when it appears, and the event version-released. - Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.VersionsBucket, "no released version is kept, so no package-published makes a version", logger.Printf)}, - BuildEdges: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.BuildEdgesBucket, "no build of another watched branch keeps its edges", logger.Printf)}, + Versions: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.VersionsBucket, "no released version is kept, so no package-published makes a version", logger.Printf)}, + BuildEdges: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.BuildEdgesBucket, "no build of another watched branch keeps its edges", logger.Printf)}, + Pins: pinsStore, PinsPending: pendingOf(js, cred), Inputs: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.InputsBucket, "no input of a run ask is kept, so the verb input answers none", logger.Printf)}, SayVersionReleased: sayVersionReleased(conn, cred), SayVersionsFilled: sayVersionsFilled(conn, cred), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; @@ -682,6 +686,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := hub.Handle(meshdelivery.SubjectSourceMovedAll, func(m jetstream.Msg) { source.heard(ctx, m) }); err != nil { return err } + pinsLedger := process.NewConsumerLedger(process.EventsStream, cred.EventConsumerName(), consumerSink(consumers), nil, logger.Printf) + for _, subject := range pinsEvents { + if err := hub.Handle(subject, func(m jetstream.Msg) { heardPins(ctx, ix, pinsLedger, m, logger.Printf) }); err != nil { + return err + } + } + // Once the state is bound the pins are made and every kept revision judged (§579: at its start). + pinsBucket.Ready(func(jetstream.KeyValue) { + go func() { + if err := ix.PinsMade(ctx); err != nil { + logger.Printf("pins: the state %s is not made: %v", index.PinsState, err) + return + } + ix.JudgePins(ctx) + }() + }) versions := func(m jetstream.Msg) { if !heardVersion(ctx, ix, m, logger.Printf) { _ = m.Ack() // nothing else the indexer consumes rides the shared stream diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go new file mode 100644 index 00000000..98e250f0 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/pins.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "encoding/json" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-indexer/internal/index" +) + +// The indexer's durable consumer of mesh-delivery's pinned, freed, heads-replaced and pins-stated (design 47 G §153, §216, §579): +// each event is applied to the state pins, and only then acknowledged, so one said while the indexer was down is applied when it +// comes back. The consumer's facts are kept in the state consumers (A-176) for indexer_consumer. + +// pinsEvents are the subjects the indexer applies to its pins. +var pinsEvents = []string{meshdelivery.SubjectPinned, meshdelivery.SubjectFreed, meshdelivery.SubjectHeadsReplaced, meshdelivery.SubjectPinsStated} + +// pendingOf says how many messages of the module's durable consumer of the events stream have not been given to it yet. +func pendingOf(js jetstream.JetStream, cred process.Account) func(context.Context) (uint64, error) { + return func(ctx context.Context) (uint64, error) { + cons, err := cred.EventConsumer(ctx, js, process.EventsStream) + if err != nil { + return 0, err + } + info, err := cons.Info(ctx) + if err != nil { + return 0, err + } + return info.NumPending, nil + } +} + +// consumerSink writes one key of the state consumers once its bucket is bound. +func consumerSink(b *process.AwaitedBucket) process.ConsumerSink { + return func(ctx context.Context, key string, value []byte) error { + kv := b.KV() + if kv == nil { + return process.ErrNotYet + } + _, err := kv.Put(ctx, key, value) + return err + } +} + +// heardPins applies one pins event, acknowledges it, and judges the revisions kept: an event not applied is left unacknowledged, so +// the bus gives it again. +func heardPins(ctx context.Context, ix *index.Indexer, ledger *process.ConsumerLedger, m jetstream.Msg, logf func(string, ...any)) { + ledger.Delivered(m) + if err := ix.HeardPins(ctx, m.Subject(), m.Data()); err != nil { + logf("pins: %s is not applied, left for the bus to give again: %v", m.Subject(), err) + return + } + _ = m.Ack() + var body struct { + At time.Time `json:"at"` + } + _ = json.Unmarshal(m.Data(), &body) + ledger.Applied(m, body.At) + ix.JudgePins(ctx) +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 552b9afb..c61356e1 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -28,6 +28,7 @@ type takerIn interface { RecomputeByHand(ctx context.Context, by, why string) index.TakenIn DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn + Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn } @@ -58,6 +59,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.NamedFingerprint != "" && a.Drop { asked, r = "drop of a member of a fingerprint's names", ix.NamedDrop(ctx, a.NamedFingerprint, a.NamedMember, a.By, a.Why) a.Module = r.Module + } else if a.UnpinRevision != "" { + asked, r = "unpin of a record from a catalog graph revision", ix.Unpin(ctx, a.UnpinRevision, a.UnpinRecord, a.By, a.Why) + a.Module = r.Module } else if a.Fingerprints { asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module) } else if a.RecoverTries { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index cce9eba5..f86ae3f7 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -181,11 +181,15 @@ type Indexer struct { saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes // OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none. - OwnBuild func(ctx context.Context) (string, error) - underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) - judgeMu sync.Mutex // one judge pass at a time (verdicts.go) - ownMu sync.Mutex // guards ownFP - keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) + OwnBuild func(ctx context.Context) (string, error) + underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) + judgeMu sync.Mutex // one judge pass at a time (verdicts.go) + ownMu sync.Mutex // guards ownFP + keepMu sync.Mutex // one catalog graph revision kept at a time (keepEntries) + // Pins is the state pins (pins.go); nil keeps none. PinsPending says how many messages the indexer's durable consumer of the + // events stream has not been given: a revision is removed only while none is pending (§579). + Pins Store + PinsPending func(ctx context.Context) (uint64, error) namesMu sync.Mutex // packageNames packageNames map[string]string // the package names read through the forge, by repository, commit, path and artifact (artifacts.go) catchingUp atomic.Bool // the first start's catch-up runs: its merges keep no revision until it ends (CatchUp) diff --git a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go index cc0508d4..4134c2be 100644 --- a/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go +++ b/modules/mesh-delivery-indexer/internal/index/catalog_revisions.go @@ -348,6 +348,12 @@ func (ix *Indexer) keepBranchEntries(ctx context.Context, branch string, src int ix.logf("catalog graph: revision %d of source revision %d is not kept: %v", rev.Revision, src, err) return err } + // The revision before this one in its branch ceases to be current (§579): it goes unless a record pins it. + if removed, err := ix.judgePinsLocked(ctx); err != nil { + ix.logf("pins: the revisions no record pins are not removed: %v", err) + } else if len(removed) > 0 { + ix.logf("pins: %d revisions removed (%s), no record pins them and none is current", len(removed), strings.Join(removed, ", ")) + } if _, err := ix.removeUnlisted(ctx); err != nil { ix.logf("catalog graph: the entries nothing lists or names any more are not removed: %v", err) // at the first revision after none does (§675) } diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index db632c15..3005971b 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -910,7 +910,7 @@ func (ix *Indexer) dry() *dryRun { Requirements: nil, Findings: nil, SeatDefinitions: nil, ReadFile: ix.ReadFile, SaySeatDefinition: nil, OwnBuildState: nil, Heard: nil, Placement: nil, RunningWhole: nil, underSeats: true, // A dry run keeps no catalog graph revision (catalog_revisions.go): its revision is the take-in's to keep. - Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, + Revisions: nil, SayCatalogRevision: nil, SayFill: nil, EntryIndex: nil, NamedSets: nil, Versions: nil, SayVersionReleased: nil, SayVersionsFilled: nil, Inputs: nil, Pins: nil, PinsPending: nil, // A dry run writes no registration and asks no one. Registry: nil, SeedModules: nil} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index b7261c76..37c90f43 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "BuildEdges"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "BuildEdges", "Pins", "PinsPending"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/pins.go b/modules/mesh-delivery-indexer/internal/index/pins.go new file mode 100644 index 00000000..f41237a4 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/pins.go @@ -0,0 +1,576 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strconv" + "strings" + "time" + + mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The indexer keeps the records pinning each catalog graph revision and removes a revision only on an event (design 47 G §153, +// §178, §189, §215, §216, §579, A-169, A-171): its state pins holds one key per revision, `.` (the watched branch +// and the revision's number, the branch escaped as a name is), whose value lists the delivery record ids that pin it, and two more +// keys, `made` (written once, create-only, when the state is first used) and `pins-stated` (the `at` of the newest applied event of +// that name). Pins count only once a pins-stated later than made is applied. +const ( + // PinsState and PinsBucket are the state's name and its bucket; PinsCapMiB is its cap in the manifest (about 1,800 keys of about 0.2 KiB). + PinsState = "pins" + PinsBucket = Module + "_" + PinsState + PinsHistory = 1 + PinsCapMiB = 4 + // PinsMadeKey and PinsStatedKey are the two keys of the state that are no revision's. + PinsMadeKey = "made" + PinsStatedKey = "pins-stated" + // PinMissingKind is the finding of a pinned revision the indexer no longer holds. + PinMissingKind = "pin-missing" + // PinsTool reads the pins, UnpinTool removes one record's pin by hand. + PinsTool = "indexer_pins" + UnpinTool = "indexer_unpin" +) + +// PinValue is one revision's key: the records pinning it, and whether the revision was not held when a record pinned it. +type PinValue struct { + Records []string `json:"records"` + Missing bool `json:"missing,omitempty"` +} + +// PinStamp is the value of the keys made and pins-stated. +type PinStamp struct { + At time.Time `json:"at"` +} + +// PinKey is a revision's key in the state pins. +func PinKey(branch string, revision int64) string { + return keys.EscapeName(branch) + "." + strconv.FormatInt(revision, 10) +} + +// ParsePinKey is the branch and the revision a key of the state pins names; false for made, pins-stated and what is no revision's key. +func ParsePinKey(key string) (branch string, revision int64, ok bool) { + i := strings.LastIndex(key, ".") + if i <= 0 { + return "", 0, false + } + n, err := strconv.ParseInt(key[i+1:], 10, 64) + if err != nil { + return "", 0, false + } + b, err := keys.UnescapeName(key[:i]) + if err != nil { + return "", 0, false + } + return b, n, true +} + +// pinObject is the revision object a key names. +func pinObject(key string) (string, bool) { + branch, revision, ok := ParsePinKey(key) + if !ok { + return "", false + } + return RevisionObject(meshcatalog.CatalogRevision{Branch: branch, Revision: revision}), true +} + +func (ix *Indexer) pinsStamp(ctx context.Context, key string) (PinStamp, bool, error) { + var s PinStamp + v, _, found, err := ix.Pins.Get(ctx, key) + if err != nil || !found { + return s, false, err + } + if err := json.Unmarshal(v, &s); err != nil { + return s, false, fmt.Errorf("%s.%s: %w", PinsState, key, err) + } + return s, true, nil +} + +// PinsMade writes the key made once, create-only, when the state is first used; a made already written stays. +func (ix *Indexer) PinsMade(ctx context.Context) error { + if ix.Pins == nil { + return nil + } + data, err := json.Marshal(PinStamp{At: ix.now().UTC()}) + if err != nil { + return err + } + if err := ix.Pins.Create(ctx, PinsMadeKey, data); err != nil && !errors.Is(err, ErrConflict) { + return fmt.Errorf("%s.%s: %w", PinsState, PinsMadeKey, err) + } + return nil +} + +// pinsCounted says the pins count: a pins-stated later than made is applied. +func (ix *Indexer) pinsCounted(ctx context.Context) (bool, error) { + made, ok, err := ix.pinsStamp(ctx, PinsMadeKey) + if err != nil || !ok { + return false, err + } + stated, ok, err := ix.pinsStamp(ctx, PinsStatedKey) + if err != nil || !ok { + return false, err + } + return stated.At.After(made.At), nil +} + +// HeardPins applies one event of the seat mesh-delivery the indexer's durable consumer gave it: pinned, freed, heads-replaced or +// pins-stated, by its subject. An error leaves the event unacknowledged, so the bus gives it again; one applied twice changes nothing. +func (ix *Indexer) HeardPins(ctx context.Context, subject string, body []byte) error { + if ix.Pins == nil { + return errors.New("the indexer keeps no state " + PinsState) + } + if err := ix.PinsMade(ctx); err != nil { + return err + } + switch subject { + case meshdelivery.SubjectPinned, meshdelivery.SubjectFreed: + var p meshdelivery.Pins + if err := json.Unmarshal(body, &p); err != nil { + return fmt.Errorf("%s is not a pins body: %w", subject, err) + } + for _, c := range p.Catalog { + key := PinKey(c.Branch, c.Revision) + if subject == meshdelivery.SubjectPinned { + if err := ix.pin(ctx, key, p.Record); err != nil { + return err + } + } else if err := ix.unpin(ctx, key, p.Record); err != nil { + return err + } + } + case meshdelivery.SubjectHeadsReplaced: + var h meshdelivery.HeadsReplacedEvent + if err := json.Unmarshal(body, &h); err != nil { + return fmt.Errorf("%s is not a heads-replaced body: %w", subject, err) + } + ix.DropCandidateGraph(ctx, h.Group, h.Heads, "mesh-delivery", "its event heads-replaced: the candidate's heads were replaced or its pull request closed") + case meshdelivery.SubjectPinsStated: + var s meshdelivery.PinsStatedEvent + if err := json.Unmarshal(body, &s); err != nil { + return fmt.Errorf("%s is not a pins-stated body: %w", subject, err) + } + err := write(ctx, ix.Pins, PinsStatedKey, func(old []byte, found bool) ([]byte, bool, error) { + var cur PinStamp + if found && json.Unmarshal(old, &cur) == nil && !s.At.After(cur.At) { + return nil, false, nil + } + data, err := json.Marshal(PinStamp{At: s.At.UTC()}) + return data, false, err + }) + if err != nil { + return err + } + default: + return fmt.Errorf("%s is no event of the pins", subject) + } + return nil +} + +// pin adds a record's id to a revision's key by compare-and-set; a revision not held is marked missing and said. +func (ix *Indexer) pin(ctx context.Context, key, record string) error { + object, _ := pinObject(key) + missing := false + if ix.Revisions != nil { + if _, err := ix.Revisions.Get(ctx, object); errors.Is(err, ErrNoObject) { + missing = true + } else if err != nil { + return fmt.Errorf("%s cannot be read: %w", object, err) + } + } + err := write(ctx, ix.Pins, key, func(old []byte, found bool) ([]byte, bool, error) { + var v PinValue + if found { + if err := json.Unmarshal(old, &v); err != nil { + return nil, false, fmt.Errorf("%s.%s: %w", PinsState, key, err) + } + } + if !slices.Contains(v.Records, record) { + v.Records = append(v.Records, record) + slices.Sort(v.Records) + } + v.Missing = v.Missing || (missing && !found) + data, err := json.Marshal(v) + return data, false, err + }) + if err != nil { + return err + } + if missing { + ix.sayPinMissing(ctx, key, record) + } + return nil +} + +// unpin removes a record's id from a revision's key; a missing revision's key left with no record goes, and its finding. +func (ix *Indexer) unpin(ctx context.Context, key, record string) error { + var gone bool + err := write(ctx, ix.Pins, key, func(old []byte, found bool) ([]byte, bool, error) { + gone = false + if !found { + return nil, false, nil + } + var v PinValue + if err := json.Unmarshal(old, &v); err != nil { + return nil, false, fmt.Errorf("%s.%s: %w", PinsState, key, err) + } + v.Records = slices.DeleteFunc(v.Records, func(r string) bool { return r == record }) + if v.Missing && len(v.Records) == 0 { + gone = true + return nil, true, nil + } + data, err := json.Marshal(v) + return data, false, err + }) + if err == nil && gone { + ix.clearPinMissing(ctx, key) + } + return err +} + +func pinFindingID(key string) string { return Module + ".pin." + key } + +func (ix *Indexer) sayPinMissing(ctx context.Context, key, record string) { + if ix.Findings == nil { + return + } + object, _ := pinObject(key) + f := ix.observed(mcc.ScopeModule, pinFindingID(key), PinMissingKind, + fmt.Sprintf("the delivery record %s pins the catalog graph revision %s, which the indexer no longer holds", record, object), + "A delivery record points at a catalog graph revision the indexer does not keep", + "The indexer removed the revision, or never kept it, before the record pinned it. indexer_unpin removes the record's pin and clears this.", PinMissingKind) + if err := ix.Findings.Write(ctx, f); err != nil { + ix.logf("pins: the finding %s for %s is not written: %v", PinMissingKind, key, err) + } +} + +func (ix *Indexer) clearPinMissing(ctx context.Context, key string) { + if ix.Findings == nil { + return + } + if err := ix.clearFinding(ctx, mcc.ScopeModule, pinFindingID(key), PinMissingKind, "no record pins "+key+" any more"); err != nil { + ix.logf("pins: the finding %s for %s is not cleared: %v", PinMissingKind, key, err) + } +} + +// RevisionGroup says which watched branch's line a revision object belongs to, and its number; false for a candidate or an entry. +func RevisionGroup(name string) (group string, revision int64, ok bool) { + if n, ok := TrunkRevisionOf(name); ok { + return "", n, true + } + if rest, ok := strings.CutPrefix(name, "branch."); ok { + i := strings.LastIndex(rest, ".") + if i > 0 { + if n, err := strconv.ParseInt(rest[i+1:], 10, 64); err == nil { + return rest[:i], n, true + } + } + } + return "", 0, false +} + +// CurrentRevisions are the kept revision objects that are current (§579): the newest of each watched branch, and every open +// candidate's, with why each is. +func CurrentRevisions(names []string) map[string]string { + newest := map[string]int64{} + for _, n := range names { + if g, r, ok := RevisionGroup(n); ok && r > newest[g] { + newest[g] = r + } + } + out := map[string]string{} + for _, n := range names { + if g, r, ok := RevisionGroup(n); ok && r == newest[g] { + out[n] = "the newest revision of its watched branch" + } else if strings.HasPrefix(n, "candidate.") { + out[n] = "an open candidate's" + } + } + return out +} + +// JudgePins removes every kept revision no record pins and which is not current, entries no kept revision lists with them, and +// says room; at the start, when the consumer is caught up and the pins count, it judges every kept revision (§579). It judges on +// applying freed, on a revision ceasing to be current and on being caught up, never on a timer. +func (ix *Indexer) JudgePins(ctx context.Context) { + if ix.Pins == nil || ix.Revisions == nil { + return + } + ix.keepMu.Lock() + removed, err := ix.judgePinsLocked(ctx) + var dropped []string + if err == nil && len(removed) > 0 { + dropped, err = ix.removeUnlisted(ctx) + } + ix.keepMu.Unlock() + if err != nil { + ix.logf("pins: the revisions no record pins are not removed: %v", err) + } + if len(removed) > 0 { + ix.logf("pins: %d revisions removed (%s) with %d entries, no record pins them and none is current", len(removed), strings.Join(removed, ", "), len(dropped)) + if _, err := ix.RoomAgain(ctx); err != nil { + ix.logf("pins: room is not evaluated: %v", err) + } + } +} + +// judgePinsLocked removes the revisions; the caller holds keepMu. It removes none while the consumer is not caught up or the pins do +// not count yet. +func (ix *Indexer) judgePinsLocked(ctx context.Context) ([]string, error) { + if ix.PinsPending == nil { + return nil, nil + } + pending, err := ix.PinsPending(ctx) + if err != nil || pending != 0 { + return nil, err + } + counted, err := ix.pinsCounted(ctx) + if err != nil || !counted { + return nil, err + } + names, err := ix.Revisions.Names(ctx) + if err != nil { + return nil, err + } + current := CurrentRevisions(names) + pinned := map[string]bool{} + pinKeys, err := ix.Pins.Keys(ctx) + if err != nil { + return nil, err + } + for _, k := range pinKeys { + object, ok := pinObject(k) + if !ok { + continue + } + v, _, found, err := ix.Pins.Get(ctx, k) + if err != nil { + return nil, err + } + var pv PinValue + if found && json.Unmarshal(v, &pv) == nil && len(pv.Records) > 0 { + pinned[object] = true + } + } + var removed []string + for _, n := range names { + if _, _, ok := RevisionGroup(n); !ok || pinned[n] { + continue + } + if _, cur := current[n]; cur { + continue + } + if err := ix.Revisions.Delete(ctx, n); err != nil { + return removed, fmt.Errorf("%s: %w", n, err) + } + removed = append(removed, n) + } + for _, k := range pinKeys { + object, ok := pinObject(k) + if !ok || !slices.Contains(removed, object) { + continue + } + if v, rev, found, err := ix.Pins.Get(ctx, k); err == nil && found { + var pv PinValue + if json.Unmarshal(v, &pv) == nil && len(pv.Records) == 0 { + _ = ix.Pins.Delete(ctx, k, rev) + } + } + } + sort.Strings(removed) + return removed, nil +} + +// Unpin removes one record's pin from a revision by hand (indexer_unpin), for a freed that was lost, and the removal rule then applies. +func (ix *Indexer) Unpin(ctx context.Context, revision, record, by, why string) TakenIn { + r := TakenIn{Module: "pins " + revision} + if ix.Pins == nil { + r.Answer = "the indexer keeps no state " + PinsState + ": nothing unpinned" + return r + } + if _, _, ok := ParsePinKey(revision); !ok { + r.Answer = revision + " is no key of the state " + PinsState + " as indexer_pins names them: nothing unpinned" + return r + } + if err := ix.unpin(ctx, revision, record); err != nil { + r.Answer = "the pin of " + record + " on " + revision + " could not be removed: " + err.Error() + ix.logf("%s", r.Answer) + return r + } + r.Stored = true + r.Answer = fmt.Sprintf("the pin of %s on %s is removed by %s (%s)", record, revision, by, why) + ix.logf("%s", r.Answer) + ix.JudgePins(ctx) + return r +} + +// PinnedRevision is one revision of the state pins as indexer_pins lists it. +type PinnedRevision struct { + Revision string `json:"revision"` + Object string `json:"object"` + Records []string `json:"records"` + Missing bool `json:"missing,omitempty"` + Kept bool `json:"kept"` + // Current says it is current, Why in what way. + Current bool `json:"current,omitempty"` + Why string `json:"why,omitempty"` +} + +// PinsAnswer is what indexer_pins answers. +type PinsAnswer struct { + Pins []PinnedRevision `json:"pins"` + // Unpinned are the kept revisions no record pins, current or not, and Current why each current one is. + Unpinned []string `json:"unpinned,omitempty"` + Made *time.Time `json:"made,omitempty"` + Stated *time.Time `json:"pins-stated,omitempty"` + // Counted says the pins count: a pins-stated later than made is applied. + Counted bool `json:"counted"` + // Fill is the state's bytes against its cap. + Fill meshcatalog.Fill `json:"fill"` +} + +// ReadPins reads the state pins and the revisions kept, for indexer_pins. +func ReadPins(ctx context.Context, pins Store, o Objects) (PinsAnswer, error) { + ix := &Indexer{Pins: pins, Revisions: o} + var out PinsAnswer + names, err := o.Names(ctx) + if err != nil { + return out, fmt.Errorf("the catalog graph revisions cannot be read: %w", err) + } + current := CurrentRevisions(names) + pinnedObjects := map[string]bool{} + pinKeys, err := pins.Keys(ctx) + if err != nil { + return out, fmt.Errorf("the state %s cannot be read: %w", PinsState, err) + } + slices.Sort(pinKeys) + var bytes int64 + for _, k := range pinKeys { + v, _, found, err := pins.Get(ctx, k) + if err != nil || !found { + continue + } + bytes += int64(len(k) + len(v)) + if object, ok := pinObject(k); ok { + var pv PinValue + if err := json.Unmarshal(v, &pv); err != nil { + return out, fmt.Errorf("%s.%s: %w", PinsState, k, err) + } + why, cur := current[object] + p := PinnedRevision{Revision: k, Object: object, Records: pv.Records, Missing: pv.Missing, Kept: slices.Contains(names, object), Current: cur, Why: why} + if len(pv.Records) > 0 { + pinnedObjects[object] = true + } + out.Pins = append(out.Pins, p) + } + } + for _, n := range names { + if _, _, ok := RevisionGroup(n); ok && !pinnedObjects[n] { + out.Unpinned = append(out.Unpinned, n) + } + } + if s, ok, err := ix.pinsStamp(ctx, PinsMadeKey); err == nil && ok { + out.Made = &s.At + } + if s, ok, err := ix.pinsStamp(ctx, PinsStatedKey); err == nil && ok { + out.Stated = &s.At + } + out.Counted, _ = ix.pinsCounted(ctx) + out.Fill = meshcatalog.Fill{Bytes: bytes, Cap: PinsCapMiB << 20, Store: PinsState} + return out, nil +} + +// PinsTools are indexer_pins (class read) and indexer_unpin (class act: an ask in takeins the indexer, the state's owner, applies; +// record says the hand act in the hand-act log before it is asked). +func PinsTools(pins Store, o Objects, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { + return []stdio.Tool{ + {Name: PinsTool, + Description: "Every catalog graph revision a delivery record pins in the indexer's state pins (design 47 G §579): the revision's " + + "key `.`, the records pinning it, whether it was missing when pinned (finding pin-missing), whether it is " + + "kept and whether it is current and why; the kept revisions no record pins; when the state was made and when the newest " + + "pins-stated was applied, and whether pins count yet (a pins-stated later than made); and the state's fill against its cap. " + + "indexer_consumer reads the consumer the events come through; indexer_unpin removes one record's pin.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return ReadPins(context.Background(), pins, o) + }}, + {Name: UnpinTool, + Description: "Remove one record's pin from one revision of the indexer's state pins, for a freed that was lost; the removal rule " + + "then applies (a revision no record pins and not current goes). A hand act, recorded through mesh-hand-acts before it is asked. " + + "Clears the finding pin-missing of a missing revision left with no record. indexer_pins names the revisions and records.", + Input: map[string]any{"type": "object", "required": []string{"revision", "record", "why"}, "properties": map[string]any{ + "revision": map[string]any{"type": "string", "description": "the revision's key `.`, as indexer_pins names it"}, + "record": map[string]any{"type": "string", "description": "the delivery record's id"}, + "why": map[string]any{"type": "string", "description": "why it is unpinned, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + revision, rec, why := str("revision"), str("record"), str("why") + if _, _, ok := ParsePinKey(revision); !ok || rec == "" || why == "" { + return nil, errors.New("name the revision as indexer_pins names it (revision) and the record (record) and say why (why): an unpin is a hand act, recorded through mesh-hand-acts") + } + data, err := json.Marshal(TakeInAsk{V: 1, UnpinRevision: revision, UnpinRecord: rec, Why: why, By: stdio.CallerOf(args), At: now().UTC()}) + if err != nil { + return nil, err + } + if err := record("unpinned the record "+rec+" from the catalog graph revision "+revision+" by hand", why); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was unpinned: %w", err) + } + a, err := ask("unpin."+keys.EscapeName(revision)+"."+keys.EscapeName(rec), data) + if err != nil { + return nil, err + } + return a.Result, nil + }}, + } +} + +// ConsumerTool is indexer_consumer (class read): each durable consumer of the indexer as the process keeps it in the state consumers +// (process.ConsumerLedger, A-176): its stream, name, acknowledged sequence, the messages pending, the last event applied with its +// `at`, and when the oldest message delivered and not yet applied was delivered. +const ConsumerTool = "indexer_consumer" + +// ConsumersAnswer is what indexer_consumer answers. +type ConsumersAnswer struct { + Consumers []process.ConsumerFacts `json:"consumers"` +} + +// ReadConsumers reads the state consumers. +func ReadConsumers(ctx context.Context, st Store) (ConsumersAnswer, error) { + out := ConsumersAnswer{Consumers: []process.ConsumerFacts{}} + ks, err := st.Keys(ctx) + if err != nil { + return out, fmt.Errorf("the state %s cannot be read: %w", process.ConsumersState, err) + } + slices.Sort(ks) + for _, k := range ks { + v, _, found, err := st.Get(ctx, k) + if err != nil || !found { + continue + } + var f process.ConsumerFacts + if err := json.Unmarshal(v, &f); err != nil { + return out, fmt.Errorf("%s.%s: %w", process.ConsumersState, k, err) + } + out.Consumers = append(out.Consumers, f) + } + return out, nil +} + +// ConsumerTools is indexer_consumer. +func ConsumerTools(st Store) []stdio.Tool { + return []stdio.Tool{{Name: ConsumerTool, + Description: "Each durable consumer of the indexer, as its process keeps it in the state consumers after each event it applies: its stream " + + "and name, the acknowledged sequence, the messages pending, the last event applied with its time, and when the oldest message delivered " + + "and not yet applied was delivered. The pins events (mesh-delivery's pinned, freed, heads-replaced and pins-stated) come through it; " + + "indexer_pins reads what they kept.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { return ReadConsumers(context.Background(), st) }}} +} diff --git a/modules/mesh-delivery-indexer/internal/index/pins_test.go b/modules/mesh-delivery-indexer/internal/index/pins_test.go new file mode 100644 index 00000000..11437d42 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/pins_test.go @@ -0,0 +1,121 @@ +package index + +import ( + "context" + "encoding/json" + "slices" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" +) + +// Design 47 G §579, A-169, A-171: the indexer keeps the records pinning each catalog graph revision, counts a record once, and +// removes a revision only while its consumer is caught up and after a pins-stated later than made: one two records pin stays +// until both free it, one neither pinned nor current goes, a revision pinned and not held is said, and unpin clears it. +func TestTheIndexerRemovesARevisionOnlyWhenNoRecordPinsItAndItIsNotCurrent(t *testing.T) { + ctx := context.Background() + clock := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) + pending := uint64(0) + ix := &Indexer{Revisions: newMemObjects(1 << 20), Pins: newMem(), Findings: newFindings(), Now: func() time.Time { return clock }, + PinsPending: func(context.Context) (uint64, error) { return pending, nil }} + reader := meshcatalog.CatalogEntry{Module: "reader", Repository: "novox/r", Manifest: map[string]any{}} + for _, commit := range []string{"c1", "c2", "c3"} { + e := reader + e.Commit = commit + if err := ix.KeepBranchEntries(ctx, "main", 5, map[string]meshcatalog.CatalogEntry{"reader": e}); err != nil { + t.Fatal(err) + } + } + kept := func() []string { + n, _ := ix.Revisions.Names(ctx) + return slices.DeleteFunc(n, func(s string) bool { _, _, ok := RevisionGroup(s); return !ok }) + } + say := func(subject string, body any) { + t.Helper() + raw, _ := json.Marshal(body) + if err := ix.HeardPins(ctx, subject, raw); err != nil { + t.Fatal(err) + } + } + pins := func(record string, revisions ...int64) meshdelivery.Pins { + p := meshdelivery.Pins{Record: record, Milestone: "47/G"} + for _, r := range revisions { + p.Catalog = append(p.Catalog, meshdelivery.PinsCatalogItem{Branch: "main", Revision: r}) + } + return p + } + want := func(what string, names ...string) { + t.Helper() + if got := kept(); !slices.Equal(got, names) { + t.Errorf("%s: kept %v, want %v", what, got, names) + } + } + say(meshdelivery.SubjectPinned, pins("r1", 1)) + say(meshdelivery.SubjectPinned, pins("r1", 1)) // heard twice: pins once + say(meshdelivery.SubjectPinned, pins("r2", 1)) + ix.JudgePins(ctx) + want("before a pins-stated later than made is applied", "trunk.1", "trunk.2", "trunk.3") + clock = clock.Add(time.Second) + say(meshdelivery.SubjectPinsStated, meshdelivery.PinsStatedEvent{At: clock}) + pending = 3 + ix.JudgePins(ctx) + want("while messages of the consumer are pending", "trunk.1", "trunk.2", "trunk.3") + pending = 0 + ix.JudgePins(ctx) + want("pinned by two records, the middle one neither pinned nor current", "trunk.1", "trunk.3") + say(meshdelivery.SubjectFreed, pins("r1", 1)) + ix.JudgePins(ctx) + want("one of two records freed it", "trunk.1", "trunk.3") + say(meshdelivery.SubjectFreed, pins("r2", 1)) + ix.JudgePins(ctx) + want("both freed it", "trunk.3") + if keys, _ := ix.Pins.Keys(ctx); slices.Contains(keys, "main.1") { + t.Errorf("the state holds %v, want no key of the removed revision", keys) + } + // A record pins a revision no longer held: its key says missing and the finding names it; unpin clears both. + say(meshdelivery.SubjectPinned, pins("r3", 1)) + if f := ix.Findings.Held(); len(f) != 1 || f[0].Kind != PinMissingKind { + t.Fatalf("findings %v, want one %s", f, PinMissingKind) + } + if r := ix.Unpin(ctx, "main.1", "r3", "operator", "freed was lost"); !r.Stored { + t.Fatalf("unpin answered %q", r.Answer) + } + if f := ix.Findings.Held(); len(f) != 0 { + t.Errorf("findings %v after unpin, want none", f) + } +} + +func TestAPinsStatedBeforeTheStateWasMadeCountsNothing(t *testing.T) { + ctx := context.Background() + clock := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) + ix := &Indexer{Revisions: newMemObjects(1 << 20), Pins: newMem(), Now: func() time.Time { return clock }, + PinsPending: func(context.Context) (uint64, error) { return 0, nil }} + raw, _ := json.Marshal(meshdelivery.PinsStatedEvent{At: clock.Add(-time.Hour)}) + if err := ix.HeardPins(ctx, meshdelivery.SubjectPinsStated, raw); err != nil { + t.Fatal(err) + } + if ok, err := ix.pinsCounted(ctx); err != nil || ok { + t.Errorf("counted %v, %v after a pins-stated older than made, want not counted", ok, err) + } +} + +func TestHeadsReplacedDropsTheCandidatesGraph(t *testing.T) { + ctx := context.Background() + ix := &Indexer{Revisions: newMemObjects(1 << 20), Pins: newMem()} + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Kind: "candidate", Group: "feat-x", Heads: "0123456789ab", Revision: 1}, nil) + if err != nil { + t.Fatal(err) + } + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(meshdelivery.HeadsReplacedEvent{Group: "feat-x", Heads: "0123456789ab", Repository: "novox/r", Number: 1}) + if err := ix.HeardPins(ctx, meshdelivery.SubjectHeadsReplaced, raw); err != nil { + t.Fatal(err) + } + if n, _ := ix.Revisions.Names(ctx); len(n) != 0 { + t.Errorf("the objects hold %v, want the candidate's graph dropped", n) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index f007fddb..c75ce420 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -73,7 +73,10 @@ type TakeInAsk struct { // NamedFingerprint and NamedMember, with Drop, ask one member to be removed from that fingerprint's set of the state named // (indexer_named_drop, named.go, design 47 G §675). NamedFingerprint string `json:"named-fingerprint,omitempty"` - NamedMember string `json:"named-member,omitempty"` + // UnpinRevision and UnpinRecord ask one record's pin on a revision of the state pins to be removed (indexer_unpin). + UnpinRevision string `json:"unpin-revision,omitempty"` + UnpinRecord string `json:"unpin-record,omitempty"` + NamedMember string `json:"named-member,omitempty"` // Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations, // indexer_proposal_drop or indexer_seed_again (design 47 G §3, §4, §8, §667). Registry *RegistryAct `json:"registry,omitempty"` @@ -95,7 +98,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.Registry == nil { + if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.UnpinRevision == "" && a.Registry == nil { return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try") } return a, nil diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 853cc3db..7d03ef67 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -16,6 +16,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views" "mesh-delivery-indexer/internal/index" @@ -131,7 +132,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 {Name: index.RegistrationsState, History: 1}, {Name: index.ProposalsState, History: 1, MaxMiB: 4}, - {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 + {Name: index.VersionsState, History: 1, MaxMiB: 64}, {Name: index.BuildEdgesState, History: 1}, {Name: index.InputsState, History: 1}, {Name: index.PinsState, History: 1, MaxMiB: index.PinsCapMiB}, {Name: process.ConsumersState, History: 1}}; !reflect.DeepEqual(m.State, want) { // DLV-I7, DLV-I17 t.Errorf("state = %+v, want %+v", m.State, want) } // Milestone 47/G W4.2 (design 47, Two graphs; ADR 0619 §2): the catalog graph revisions are objects in the old section, @@ -166,7 +167,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", - "mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery.freed", "mesh-delivery.heads-replaced", "mesh-delivery.pinned", "mesh-delivery.pins-stated", "mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided", "seat:mesh-module-assigner.views.assignments"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -201,8 +202,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool) // design 47 G §667, §8 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 47 || slices.Contains(m.Tools, "modules") { + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool) // design 47 G §667, §8, §579 + if !reflect.DeepEqual(m.Tools, want) || len(want) != 50 || slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index b155f070..306fa37c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -73,7 +73,10 @@ "indexer_proposals", "indexer_proposal_drop", "indexer_seed_again", - "indexer_inputs" + "indexer_inputs", + "indexer_pins", + "indexer_unpin", + "indexer_consumer" ], "consumes": [ "go-package-registry.published", @@ -92,6 +95,10 @@ "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery.source-moved", + "mesh-delivery.pinned", + "mesh-delivery.freed", + "mesh-delivery.heads-replaced", + "mesh-delivery.pins-stated", "mesh-wants.wanted", "node-engine.running", "operator-channel.decided", @@ -123,7 +130,9 @@ { "name": "proposals", "history": 1, "max-mib": 4 }, { "name": "versions", "history": 1, "max-mib": 64 }, { "name": "build-edges", "history": 1 }, - { "name": "inputs", "history": 1 } + { "name": "inputs", "history": 1 }, + { "name": "pins", "history": 1, "max-mib": 4 }, + { "name": "consumers", "history": 1 } ], "streams": [ { "name": "registration-history", "events": ["registration.*"], "retention": "history", "max-mib": 64 }, From 3c38f9c46809c85e64760153b3c4a706c36d9e8a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:48:20 +0100 Subject: [PATCH 04/24] IX31: the whole test judges the indexer's manifest against a seat list holding the assigner seat's view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indexer consumes the seat mesh-module-assigner's views.assignments in the seat's form, which the validator accepts only where the seat list holds the seat and offers the state as a view; TestIndexerWhole seeds that definition. Builds: design 47 G §616, §699, A-119, A-131 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-indexer/integration_test.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 4aa1dec0..286fe228 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -100,6 +100,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 t.Fatal(err) } } + seedAssignerSeat(t, ctx, js) // The take-in events are the indexer's ordinary events (ADR 0633 §5), on the path manifest-accepted takes. tc, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.ToolchainsBucket("mesh-build-go"), History: 1}) if err != nil { @@ -787,6 +788,25 @@ func seedBuildModules(t *testing.T, ctx context.Context, js jetstream.JetStream, } } +// seedAssignerSeat keeps the trunk definition of the seat mesh-module-assigner, which offers its state assignments as a view (§616, +// §699): the indexer consumes it in the seat's form, and its own manifest is judged against the seat list it holds. +func seedAssignerSeat(t *testing.T, ctx context.Context, js jetstream.JetStream) { + t.Helper() + defs, err := js.KeyValue(ctx, index.SeatDefinitionsBucket) + if err != nil { + t.Fatal(err) + } + assignerDef := json.RawMessage(`{"format":1,"seat":"mesh-module-assigner","scope":"mesh","grain":"whole","domain":"Delivery","decision":"novox/hq ADR 0567",` + + `"verbs":[{"name":"assignments","input":{"type":"object"},"answer":{"type":"object"}}],"states":[{"name":"assignments","template":"state","view":true,` + + `"key-forms":[{"key":".node","value":{"type":"object"}}]}]}`) + assignerSum := sha256.Sum256(assignerDef) + assignerEntry, _ := json.Marshal(index.SeatDefinitionEntry{Seat: "mesh-module-assigner", Source: index.SourceTrunk, Repository: "novox/mesh-contracts", Module: "mesh-contracts", + Directory: ".", Commit: "4444444444444444444444444444444444444444", Digest: hex.EncodeToString(assignerSum[:]), Definition: assignerDef}) + if _, err := defs.Put(ctx, "mesh-module-assigner.trunk", assignerEntry); err != nil { + t.Fatal(err) + } +} + // startIndexer is startIndexerRun for a test that never starts the program again. func startIndexer(t *testing.T, put func(context.Context, buildLog), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, revisionView) { js, ctx, candidates, modules, _ := startIndexerRun(t, put, before...) From 0772e6dc23ebf55c67f96251f808cd8e5e24c914 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:50:15 +0100 Subject: [PATCH 05/24] IX14: a run on request's candidate from the runner's run-started, dropped once every member's pipeline ended MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indexer hears the runner's run-started and pipeline-ended on their own streams and writes candidates.run-: one pull per member, its repository and directory from its registration, its manifest and fingerprint from a take-in of its commit read through the seat git, over its branch's revision view when its registration watches the branch, else the trunk's. The state runs keeps each run's members and ended modules, so a restart between two ends still drops the candidate. The try candidates, the checker's checks consumer, checker_tries and the three try tools go; IsTryGroup takes run- beside try-. Builds: design 47 G §64, §190, §321, §66, A-160, A-174 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../go/clients/meshcatalogclient/candidate.go | 7 +- .../cmd/mesh-delivery-indexer-tools/main.go | 1 - .../mesh-delivery-indexer/integration_test.go | 139 +++--- .../cmd/mesh-delivery-indexer/main.go | 28 +- .../cmd/mesh-delivery-indexer/named.go | 29 -- .../cmd/mesh-delivery-indexer/named_test.go | 53 ++- .../cmd/mesh-delivery-indexer/previews.go | 82 +--- .../cmd/mesh-delivery-indexer/runs.go | 77 +++ .../cmd/mesh-delivery-indexer/takein.go | 8 - modules/mesh-delivery-indexer/go.mod | 4 +- .../internal/index/candidates.go | 15 +- .../internal/index/control_records.go | 2 +- .../internal/index/pending_test.go | 2 +- .../internal/index/pending_tools.go | 95 ---- .../internal/index/previews.go | 5 +- .../internal/index/previews_test.go | 219 --------- .../internal/index/runcandidates.go | 296 ++++++++++++ .../internal/index/runcandidates_test.go | 130 ++++++ .../internal/index/takein.go | 8 +- .../internal/index/trycandidates.go | 437 ------------------ .../mesh-delivery-indexer/manifest_test.go | 12 +- modules/mesh-delivery-indexer/module.json | 11 +- 22 files changed, 652 insertions(+), 1008 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go create mode 100644 modules/mesh-delivery-indexer/internal/index/runcandidates.go create mode 100644 modules/mesh-delivery-indexer/internal/index/runcandidates_test.go delete mode 100644 modules/mesh-delivery-indexer/internal/index/trycandidates.go diff --git a/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/candidate.go b/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/candidate.go index e609af0b..27a90dc8 100644 --- a/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/candidate.go +++ b/modules/mesh-delivery-contracts/go/clients/meshcatalogclient/candidate.go @@ -62,10 +62,11 @@ func ParseCandidatesKey(key string) (string, error) { return group, nil } -var tryGroupOK = regexp.MustCompile(`^try-[0-9]{1,20}$`) +var tryGroupOK = regexp.MustCompile(`^(try-[0-9]{1,20}|run-[A-Za-z0-9_-]{1,64})$`) -// IsTryGroup says a candidates key is the group of a whole try, try-, which is also the try's id: the indexer makes the -// candidate of a try's group from its previews, and the checker judges it by its try, never as a pull request's check. +// IsTryGroup says a candidates key is the group of a whole try, try-, which is also the try's id, or of a run on +// request, run- (design 47 G A-160): the indexer makes the candidate of a run from the runner's run-started, and the +// checker judges neither as a pull request's check. func IsTryGroup(key string) bool { return tryGroupOK.MatchString(key) } // NoneNoPreview is why a candidate's touched module has no fingerprint of the new form yet: its merge preview is not diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index e1736091..9248d6e3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -309,7 +309,6 @@ func main() { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.FingerprintsAgainTool, Verb: "fingerprints-again"}) return err }, 10*time.Second)...) - tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, asks, time.Now)...) tools = append(tools, configReadersTool(stdio.NewWatchedStates(), trunk.State("modules")), busStateTool(trunk.State), requirementsTool(keptSource), packageUsersTool(trunk.State), seatDefinitionTool(keptSource), outOfStepTool(keptSource)) if err := stdio.Serve("", tools); err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 21b60028..ddf6699a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -33,9 +33,10 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" @@ -1111,86 +1112,98 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes } } -// TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBucket: the indexer starts with the stream PREVIEWS -// and its durable consumer (made by the controller from the checker's declaration and the indexer's consumes) and no bucket -// mesh-delivery-checker_previews; a pull request of mesh-catalog touching avahi makes the candidate with the reason -// 'no merge preview is computed'; the checker then makes its bucket, writes the preview of that head onto the trunk head -// the source names, and says so on its stream; the candidate's avahi gets, within 20 s and with no restart, the reason -// computed from the preview (the toolchain is not reported here, so 'toolchain not yet reported'). -func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBucket(t *testing.T) { // 1445/AC4 +// TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded (design 47 G §64, §67 test 64, §190; A-160, A-174): +// the indexer starts with the runner's streams run-started and pipeline-ended and its durables on them, its state runs and a +// registration of avahi; the runner says run-started of run 9 with avahi at a commit on a branch no registration watches; +// within 20 s candidates.run-9 holds one pull of number 0 at that commit, avahi's manifest read through the seat git; the +// indexer is restarted, the runner says avahi's pipeline of run 9 ended, and the candidate and the run's key go. +func TestIndexerBuildsARunsCandidateAndDropsItWhenEveryMembersPipelineEnded(t *testing.T) { manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") if err != nil { t.Fatal(err) } const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" - const trunk, head = "5555555555555555555555555555555555555555", "0123456789abcdef0123456789abcdef01234567" - js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds buildLog) { - builds.end(t, "novox/mesh-catalog", "avahi", "modules/avahi") - }, func(ctx context.Context, js jetstream.JetStream) { - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: PreviewsStream, Subjects: []string{previewWrittenSubject}}); err != nil { - t.Fatal(err) - } - if _, err := js.CreateOrUpdateConsumer(ctx, PreviewsStream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { - t.Fatal(err) - } - seedBuildModules(t, ctx, js, nil) - // The seat mesh-delivery's verb source, as the gate serves it (design 47 G §115): main names the catalogue's trunk head. - answer := func(m *nats.Msg) { - raw, _ := json.Marshal(map[string]any{"result": meshdelivery.SourceAnswer{Branch: "main", Group: "g", Revision: 5, - Repositories: []meshdelivery.SourceHead{{Repository: "novox/mesh-catalog", Head: trunk, Trunk: "main"}}}, "node": "n1"}) - _ = m.Respond(raw) - } - for _, subject := range []string{"mesh.seat.mesh-delivery.call.source.>", "mesh.seat.mesh-delivery.tool.source"} { - if _, err := js.Conn().Subscribe(subject, answer); err != nil { + const commit = "0123456789abcdef0123456789abcdef01234567" + var runs jetstream.KeyValue + js, ctx, candidates, _, restart := startIndexerRun(t, func(context.Context, buildLog) {}, func(ctx context.Context, js jetstream.JetStream) { + for stream, subject := range map[string]string{meshpipelinerunner.StreamRunStarted: meshpipelinerunner.SubjectRunStartedAll, + meshpipelinerunner.StreamPipelineEnded: meshpipelinerunner.SubjectPipelineEndedAll} { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{subject}}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy, + FilterSubjects: []string{subject}}); err != nil { t.Fatal(err) } } + var err error + if runs, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.RunsBucket, History: 1}); err != nil { + t.Fatal(err) + } + regs, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.RegistrationsBucket, History: 1}) + if err != nil { + t.Fatal(err) + } + reg, _ := json.Marshal(meshcatalog.Registration{V: 1, Module: "avahi", Origin: "novox/mesh-catalog", Path: "modules/avahi", Branches: []string{"main"}}) + if _, err := regs.Create(ctx, index.RegistrationKey("avahi"), reg); err != nil { + t.Fatal(err) + } + seedBuildModules(t, ctx, js, nil) + // The seat git as the forge module serves it: avahi's directory and its manifest at the run's commit. + reply := func(m *nats.Msg, result any) { + raw, _ := json.Marshal(map[string]any{"result": result, "node": "n0"}) + _ = m.Respond(raw) + } + if _, err := js.Conn().Subscribe("mesh.seat.git.tool.trees", func(m *nats.Msg) { + reply(m, git.TreesAnswer{Root: "r0", Commit: commit, Trees: map[string]string{"modules/avahi": "newtree"}}) + }); err != nil { + t.Fatal(err) + } + if _, err := js.Conn().Subscribe("mesh.seat.git.tool.file", func(m *nats.Msg) { + var a git.FileInput + _ = json.Unmarshal(m.Data, &a) + switch a.Path { + case "modules/avahi/module.json": + reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: string(manifest)}) + case "modules/avahi/go.sum": + reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Content: goSum}) + default: + reply(m, git.FileAnswer{Path: a.Path, Ref: a.Ref, Absent: true}) + } + }); err != nil { + t.Fatal(err) + } }) - pull, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: forgemsg.PullUpdated, - Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": string(manifest)}}) - if err != nil { + started, _ := json.Marshal(meshpipelinerunner.RunStarted{V: 1, Run: "9", By: "lead@n1", Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: commit, Branch: "feature/1966-x"}}}) + if _, err := js.Publish(ctx, meshpipelinerunner.SubjectRunStarted("avahi", "feature-1966-x", commit), started); err != nil { t.Fatal(err) } - if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.pull", pull); err != nil { - t.Fatal(err) - } - none2 := func(want string) string { - var got string - for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - if e, err := candidates.Get(ctx, "feat/x"); err == nil { - if c, err := index.DecodeCandidate(e.Value()); err == nil && len(c.Pulls) == 1 { - got = c.Pulls[0].Fps["avahi"].None2 - if got == want { - break - } - } + key, _ := meshcatalogclient.CandidatesKey("run-9") + var c meshcatalog.CandidatesByGroupValue + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := candidates.Get(ctx, key); err == nil { + if c, err = index.DecodeCandidate(e.Value()); err == nil { + break } } - return got } - if got := none2("no merge preview is computed"); got != "no merge preview is computed" { - t.Fatalf("none2 = %q within 20 s of the pull request, want 'no merge preview is computed'", got) + if len(c.Pulls) != 1 || c.Pulls[0].Number != 0 || c.Pulls[0].Head != commit || c.Pulls[0].Fps["avahi"].Manifest != meshcatalogclient.ManifestDigest(manifest) { + t.Fatalf("candidates.run-9 = %+v within 20 s of run-started, want avahi at the run's commit with its manifest", c) } - // The checker's bucket is made only now, as when the checker is assigned after the indexer started. - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: index.PreviewsBucket, History: index.PreviewsHistory}) - if err != nil { + restart() + ended, _ := json.Marshal(meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "feature-1966-x.0.avahi", Module: "avahi", Branch: "feature/1966-x", Commit: commit, + Outcome: "passed", Steps: []meshpipelinerunner.StepResult{}, Run: "9"}) + if _, err := js.Publish(ctx, meshpipelinerunner.SubjectPipelineEnded("avahi", "feature-1966-x", commit), ended); err != nil { t.Fatal(err) } - pv := meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue{Repository: "novox/mesh-catalog", Head: head, Trunk: trunk, Merge: "6666666666666666666666666666666666666666", - Modules: map[string]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{"modules/avahi": {Tree: "newtree", Manifest: meshcatalogclient.ManifestDigest(manifest), Locks: map[string]string{"go.sum": goSum}}}} - key, _ := index.PreviewKey(pv.Repository, pv.Head, pv.Trunk) - data, err := index.EncodePreview(pv) - if err != nil { - t.Fatal(err) + gone := false + for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && !gone; time.Sleep(50 * time.Millisecond) { + _, cerr := candidates.Get(ctx, key) + _, rerr := runs.Get(ctx, "run-9") + gone = errors.Is(cerr, jetstream.ErrKeyNotFound) && errors.Is(rerr, jetstream.ErrKeyNotFound) } - if _, err := kv.Create(ctx, key, data); err != nil { - t.Fatal(err) - } - if _, err := js.Publish(ctx, previewWrittenSubject, []byte(`{"key":"`+key+`"}`)); err != nil { - t.Fatal(err) - } - if got := none2(sdkstep.ToolchainNoReport); got != sdkstep.ToolchainNoReport { - t.Errorf("none2 = %q within 20 s of the preview, want %q (computed from the preview)", got, sdkstep.ToolchainNoReport) + if !gone { + t.Error("the run's candidate or its runs key stays 20 s after its member's pipeline ended across a restart") } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 70451ace..5603ae66 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -407,7 +407,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The manifests a take-in wrote and no judge pass judged yet (issue 1966): a state this delivery brings, bound when it // appears; until then a take-in's new manifests are kept for their judging in its memory alone. unjudged := process.AwaitBucket(ctx, js, conn, index.UnjudgedBucket, "a take-in's new manifests are kept for their judging in its memory alone", logger.Printf) - tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf) + // The runs on request the indexer holds a candidate for (A-174): a state this delivery brings, bound when it appears. + runs := process.AwaitBucket(ctx, js, conn, index.RunsBucket, "no run on request's candidate is written or dropped", logger.Printf) findings := process.AwaitBucket(ctx, js, conn, findingsBucket, "no finding of the indexer is written", logger.Printf) // The indexer's findings are the seat mesh-conditions' generated finding, written through mesh-sdk's held finding writer // (design 47 G §689, A-29): it holds what is written before the state is bound, and starts once the state is. @@ -444,7 +445,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Predefined: awaitedStore{b: predefined}, ConfigCandidates: awaitedStore{b: configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), - OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph, + OwnBuild: ownBuildVia(calls.Call, cred.Node), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), Runs: awaitedStore{b: runs}, Graph: graph, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, Message: messageVia(calls.CallKept), Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) // Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go). @@ -665,17 +666,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro buildModules.hearWith(func(stream, module string) { hear(stream, module+"'s toolchain reports are not heard until the indexer starts again", func(m jetstream.Msg) { tools.Reported(m) }) }) - // The consumer on the checker's stream exists before the open tries are listed and is heard after (ADR 0638 §3): an event - // said between the call and the hearing waits in it, and a try that started or ended meanwhile is found either way. - var checksCons jetstream.Consumer - _ = notHeard.Bind(PreviewsStream, "the checker's previews and tries are not heard until the indexer starts again", func() error { - c, err := cred.EventConsumer(ctx, js, PreviewsStream) - if err != nil { - return fmt.Errorf("the consumer %s_%s on %s does not exist yet: %w", cred.Node, cred.Module, PreviewsStream, err) - } - checksCons = c - return nil - }) // The versions each package's series holds (DLV-I5): the registry seats' published, on the // indexer's durable consumer of the shared events stream. Where it is not there yet, no series moves until the indexer // starts again, said. @@ -723,6 +713,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return nil }) hearNamed(ctx, js, cred, notHeard, named, logger.Printf) + hearRuns(ctx, js, cred, notHeard, ix, logger.Printf) // The checker's verdicts carry a pull request's out-of-step files (DLV-I20): on the indexer's durable consumer of the // checker's stream asks. Where it is not there yet, no pull request's out-of-step files are kept until the indexer // starts again, said. @@ -741,17 +732,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } else if words != "" { logger.Printf("take-in at start: %s", words) } - // The open tries are found again through the checker's tool: a try event missed while the indexer was down would leave a - // try with no candidate, or a candidate with no try. Not fatal: the events come, and a preview written writes the candidates. - // A recovery that fails is said (journal, and indexer_try_candidates) and the events are heard all the same. - _ = ix.RecoverTries(ctx) - if checksCons != nil { - pc, err := process.HearEvents(checksCons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf) - if err != nil { - return err - } - defer pc.Stop() - } cons, err := cred.EventConsumer(ctx, js, forgeStream) if err != nil { // Essential: the forge's merges are what the indexer indexes. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index c99ab795..41d4a90f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/checkermsg" "path" "slices" "strings" @@ -261,31 +260,3 @@ func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, er return out, nil } } - -// openTriesTool is the checker's tool that lists the tries that wait or run (mesh-delivery-checker checker_tries). -const openTriesTool = "mesh-delivery-checker.checker_tries" - -// openTriesVia lists the open tries through the checker's tool, never its state (ADR 0596). -func openTriesVia(call toolCaller) func(ctx context.Context) ([]checkermsg.Open, error) { - return func(context.Context) ([]checkermsg.Open, error) { - raw, err := call(openTriesTool, map[string]any{}) - if err != nil { - return nil, err - } - // No try open is an answer: the checker's list is empty, or null in an older checker (issue 1921). An answer without the - // key is refused below, never read as none. - var o map[string]json.RawMessage - if err := json.Unmarshal(raw, &o); err != nil { - return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err) - } - list, ok := o["tries"] - if !ok { - return nil, fmt.Errorf("the answer of %s is not {tries}: no tries key", openTriesTool) - } - var tries []checkermsg.Open - if err := json.Unmarshal(list, &tries); err != nil { - return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err) - } - return tries, nil - } -} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index 980e96bd..52f3b9c4 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "errors" + "fmt" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" "mesh-delivery-indexer/internal/index" "slices" "strings" @@ -127,23 +129,30 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 if _, err := ownBuildVia(none, "n1")(context.Background()); err == nil { t.Error("an engine that says nothing runs gave a build") } - // Issue 1921: the checker's list of open tries; none open is an answer, whether [] or null. + // Design 47 G §64, A-160, A-174: the runner's run-started and pipeline-ended reach the indexer; one that is no such body, + // or a run that can never be taken in, is terminated, any other failure is left for redelivery, and a taken one is acked. for _, row := range []struct { - name, answer string - want int - refused bool + name, body string + ended bool + err error + acked, termd int + started, endedOf string }{ - {"one open", `{"tries":[{"id":"try-1","state":"running","whole":true,"members":[{"repository":"novox/a","head":"h","trunk":"t"}]}]}`, 1, false}, - {"an empty list", `{"tries":[]}`, 0, false}, - {"null, as an older checker answers none", `{"tries":null}`, 0, false}, - {"no tries key", `{}`, 0, true}, - {"not json", `nope`, 0, true}, - {"tries of another shape", `{"tries":{"a":1}}`, 0, true}, + {name: "a run started", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}],"by":"s"}`, acked: 1, started: "7"}, + {name: "a run's pipeline ended", body: `{"v":1,"pipeline":"p","module":"a","branch":"b","commit":"c","outcome":"passed","steps":[],"run":"7"}`, ended: true, acked: 1, endedOf: "7"}, + {name: "not json", body: `nope`, termd: 1}, + {name: "a run never taken in", body: `{"v":1,"run":"7","members":[]}`, err: fmt.Errorf("%w: no member", index.ErrUnreadable), termd: 1, started: "7"}, + {name: "a bus fault", body: `{"v":1,"run":"7","members":[{"module":"a","commit":"c"}]}`, err: errors.New("the bus did not answer"), started: "7"}, } { - call := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil } - got, err := openTriesVia(call)(context.Background()) - if (err != nil) != row.refused || len(got) != row.want { - t.Errorf("%s: %d tries, error %v; want %d, refused %v", row.name, len(got), err, row.want, row.refused) + ix := &fakeRuns{err: row.err} + m := &fakeMsg{subject: "x", data: []byte(row.body)} + if row.ended { + heardPipelineEnded(context.Background(), ix, m, t.Logf) + } else { + heardRunStarted(context.Background(), ix, m, t.Logf) + } + if m.acked != row.acked || m.termd != row.termd || ix.started != row.started || ix.ended != row.endedOf { + t.Errorf("%s: acked %d, terminated %d, started %q, ended %q", row.name, m.acked, m.termd, ix.started, ix.ended) } } } @@ -177,3 +186,19 @@ func TestReadViaReadsManifestsAndLocks(t *testing.T) { t.Fatalf("read = %+v", got) } } + +// fakeRuns records the run events handed to it and answers err. +type fakeRuns struct { + err error + started, ended string +} + +func (f *fakeRuns) StartRun(_ context.Context, e meshpipelinerunner.RunStarted) error { + f.started = e.Run + return f.err +} + +func (f *fakeRuns) EndRunPipeline(_ context.Context, e meshpipelinerunner.PipelineEnded) error { + f.ended = e.Run + return f.err +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go index a38c80b3..766e7270 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go @@ -4,30 +4,19 @@ import ( "context" "errors" "log" - "strings" "sync" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/checkermsg" - "mesh-delivery-indexer/internal/index" ) -// PreviewsStream is the stream the checker carries its events on, as the bus names it (declared by the -// checker as the stream checks, heard by this module through consumes; novox/hq ADR 0484 clause 3). -const PreviewsStream = "CHECKS" - -// previewWrittenSubject is the subject of the checker's preview.written event. -const previewWrittenSubject = "mesh.stream." + index.CheckerModule + ".event.preview.written" - // previewWatch reads the merge previews the checker writes (mesh-delivery-checker_previews) and follows them: every change // writes the candidates' fingerprints again (index.Indexer.ApplyPreview). The checker's bucket is made when it is assigned, -// after the indexer started, and the indexer holds no timer (ADR 0437 §8): it starts with the bucket absent, and the -// checker says each time it wrote a preview (preview.written, handled by Written); the bucket is then watched and settled -// as the others, with no restart. +// after the indexer started, and the indexer holds no timer (ADR 0437 §8) and hears no preview.written (design 47 G A-160): a +// bucket absent at the start is not watched until the indexer starts again. type previewWatch struct { ctx context.Context js bucketOpener @@ -53,16 +42,16 @@ func (w *previewWatch) ensure() error { kv, err := w.js.KeyValue(w.ctx, index.PreviewsBucket) switch { case errors.Is(err, jetstream.ErrBucketNotFound): - w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", index.PreviewsBucket) + w.logger.Printf("the bucket %s does not exist yet: it is watched from the indexer's next start", index.PreviewsBucket) return nil case err != nil: // Refused or not answered (its grant not on the bus yet, issue 1945): said, and opened again at the checker's next preview. - w.logger.Printf("the bucket %s cannot be read (%v): it is watched when the checker says it wrote a preview", index.PreviewsBucket, err) + w.logger.Printf("the bucket %s cannot be read (%v): it is watched from the indexer's next start", index.PreviewsBucket, err) return nil } watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) }) if err != nil { - process.NotWatched(w.logger.Printf, index.PreviewsBucket, err, "it is watched when the checker says it wrote a preview") + process.NotWatched(w.logger.Printf, index.PreviewsBucket, err, "it is watched from the indexer's next start") return nil } w.watched = true @@ -73,64 +62,5 @@ func (w *previewWatch) ensure() error { } }) }() - if err := w.ix.RecomputeCandidates(w.ctx); err != nil { - return err - } - return w.ix.WriteTryCandidates(w.ctx) -} - -// Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the -// checker's is terminated; a failure stays unacknowledged, and the server delivers it again. -func (w *previewWatch) Written(m delivered) { - if !strings.HasPrefix(m.Subject(), previewWrittenSubject) { - w.logger.Printf("%s: not the checker's preview.written, dropped", m.Subject()) - _ = m.Term() - return - } - if err := w.ensure(); err != nil { - w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err) - return - } - _ = m.Ack() -} - -// tryEventSubject is the subject of an event of the checker's stream checks about a try. -func tryEventSubject(event string) string { - return "mesh.stream." + index.CheckerModule + ".event." + event -} - -// heardChecks takes one event of the checker's stream in: a preview written (Written), or a whole try's start or end, from -// which the candidate of its group is written or deleted (index.Indexer.StartTry, EndTry). A failure stays unacknowledged, and -// the server delivers it again; a body that is no try event is terminated. -func (w *previewWatch) heardChecks(m delivered) { - var event string - switch m.Subject() { - case tryEventSubject(checkermsg.Started): - event = checkermsg.Started - case tryEventSubject(checkermsg.Ended): - event = checkermsg.Ended - default: - w.Written(m) - return - } - e, err := checkermsg.Decode(event, m.Data()) - if err != nil { - w.logger.Printf("%s: not a try event, dropped: %v", m.Subject(), err) - _ = m.Term() - return - } - if event == checkermsg.Started { - var ms []index.TryMember - for _, x := range e.Members { - ms = append(ms, index.TryMember{Repository: x.Repository, Head: x.Head, Trunk: x.Trunk}) - } - err = w.ix.StartTry(w.ctx, e.ID, ms) - } else { - err = w.ix.EndTry(w.ctx, e.ID) - } - if err != nil { - w.logger.Printf("%s of %s: left for redelivery: %v", event, e.ID, err) - return - } - _ = m.Ack() + return w.ix.RecomputeCandidates(w.ctx) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go new file mode 100644 index 00000000..6c6bdbfb --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/runs.go @@ -0,0 +1,77 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-indexer/internal/index" +) + +// runTaker is what the indexer does with the runner's events of a run on request (index.Indexer.StartRun, EndRunPipeline). +type runTaker interface { + StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error + EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error +} + +// hearRuns hears the runner's run-started and pipeline-ended (design 47 G §64, §190; A-160, A-174), each on its own stream, on +// the indexer's durable: a run's candidate is written on its start and dropped once every member's pipeline has ended. A stream +// the bus does not hold yet is the finding family-not-heard, and no run's candidate is written or dropped meanwhile. +func hearRuns(ctx context.Context, js jetstream.JetStream, cred process.Account, notHeard binder, ix runTaker, logf func(string, ...any)) { + for _, s := range []struct { + stream string + each func(jetstream.Msg) + }{ + {meshpipelinerunner.StreamRunStarted, func(m jetstream.Msg) { heardRunStarted(ctx, ix, m, logf) }}, + {meshpipelinerunner.StreamPipelineEnded, func(m jetstream.Msg) { heardPipelineEnded(ctx, ix, m, logf) }}, + } { + _ = notHeard.Bind(s.stream, "no run on request's candidate is written or dropped until the indexer starts again", func() error { + cc, err := cred.HearSeatEvents(ctx, js, s.stream, s.each, logf) + if err != nil { + return err + } + go func() { <-ctx.Done(); cc.Stop() }() + return nil + }) + } +} + +// heardRunStarted takes one run-started in. A body that is no run-started, or a run that can never be taken in, is said and +// terminated; any other failure stays unacknowledged, and the server delivers it again. +func heardRunStarted(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) { + var e meshpipelinerunner.RunStarted + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s: not a run-started, dropped: %v", m.Subject(), err) + _ = m.Term() + return + } + settleRun(m, ix.StartRun(ctx, e), "run-started of "+e.Run, logf) +} + +// heardPipelineEnded takes one pipeline-ended in: of a run's pipeline, its module is kept as ended. +func heardPipelineEnded(ctx context.Context, ix runTaker, m delivered, logf func(string, ...any)) { + var e meshpipelinerunner.PipelineEnded + if err := json.Unmarshal(m.Data(), &e); err != nil { + logf("%s: not a pipeline-ended, dropped: %v", m.Subject(), err) + _ = m.Term() + return + } + settleRun(m, ix.EndRunPipeline(ctx, e), "pipeline-ended of "+e.Module+" in the run "+e.Run, logf) +} + +func settleRun(m delivered, err error, what string, logf func(string, ...any)) { + switch { + case err == nil: + _ = m.Ack() + case errors.Is(err, index.ErrUnreadable): + logf("%s: never taken in, dropped: %v", what, err) + _ = m.Term() + default: + logf("%s: left for redelivery: %v", what, err) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 552b9afb..6e2709f1 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -22,8 +22,6 @@ type takerIn interface { TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn DropPending(ctx context.Context, key, by, why string, abandon bool) index.TakenIn TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn - DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn - RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn FingerprintsAgain(ctx context.Context, module string) index.TakenIn RecomputeByHand(ctx context.Context, by, why string) index.TakenIn DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn @@ -60,12 +58,6 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream a.Module = r.Module } else if a.Fingerprints { asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module) - } else if a.RecoverTries { - asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why) - a.Module = "tries" - } else if a.TryCandidate != "" && a.Drop { - asked, r = "drop of a try's candidate", ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why) - a.Module = a.TryCandidate } else if a.Merge != "" { asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why) a.Module = r.Module diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 9c4b866d..70c6453e 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -15,6 +15,7 @@ require ( git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 @@ -41,7 +42,6 @@ require ( replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ../mesh-delivery-builder/buildrun require ( - git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/checkermsg v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000 @@ -53,8 +53,6 @@ replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk => ../mesh-de replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein => ./takein -replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/checkermsg => ../mesh-delivery-checker/checkermsg - replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg => ../mesh-delivery-forge-bridge/forgemsg replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds => ../mesh-delivery-builder/builds diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index cce9eba5..d5bf7ef4 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -15,7 +15,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/checkermsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -215,14 +214,10 @@ type Indexer struct { Pending Store // SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none. SayTakeIn func(ctx context.Context, t takein.TakeIn) error - // OpenTries lists the tries that wait or run, from the checker's tool (checker_tries); nil recovers none (trycandidates.go). - OpenTries func(ctx context.Context) ([]checkermsg.Open, error) - // TryRecovery keeps how the last recovery of the open tries went (trycandidates.go); nil keeps none. - TryRecovery Store - tryMu sync.Mutex // one try event or recovery at a time (trycandidates.go) - recoveryFailed atomic.Bool // the last recovery of the open tries failed (trycandidates.go) - tries map[string][]TryMember // the whole tries that wait or run, by try id (trycandidates.go); guarded by mu - pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) + // Runs is the state runs: the runs on request the indexer holds a candidate for (runcandidates.go, A-174); nil keeps none. + Runs Store + runMu sync.Mutex // one run event at a time (runcandidates.go) + pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) } func (ix *Indexer) logf(format string, args ...any) { @@ -384,7 +379,7 @@ func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error { } for _, key := range keys { if meshcatalogclient.IsTryGroup(key) { - continue // a try's candidate is no pull request's: the checker's try owns it (trycandidates.go) + continue // a run's candidate is no pull request's: its run owns it (runcandidates.go) } err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) { if !found { diff --git a/modules/mesh-delivery-indexer/internal/index/control_records.go b/modules/mesh-delivery-indexer/internal/index/control_records.go index c13e960b..b5688b42 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records.go @@ -237,7 +237,7 @@ func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f forgemsg.Fo var errs []error for _, g := range groups { if meshcatalogclient.IsTryGroup(g) { - continue // a try's candidate holds no pull request's config declarations (trycandidates.go) + continue // a run's candidate holds no pull request's config declarations (runcandidates.go) } raw, _, found, err := ix.Store.Get(ctx, g) if err != nil || !found { diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index b7261c76..a732a6ff 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -1131,7 +1131,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "Unjudged", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", - "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "OpenTries", "TryRecovery", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "BuildEdges"} { + "Requirements", "Findings", "SeatDefinitions", "ReadFile", "SaySeatDefinition", "Runs", "OwnBuildState", "Heard", "Placement", "RunningWhole", "Revisions", "SayCatalogRevision", "SayFill", "EntryIndex", "NamedSets", "Registry", "SeedModules", "Versions", "SayVersionReleased", "SayVersionsFilled", "Inputs", "BuildEdges"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index 5557a43a..366654eb 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -4,12 +4,8 @@ import ( "context" "encoding/json" "fmt" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "strings" "time" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -115,94 +111,3 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T Run: merge}, } } - -// TryCandidateTools are the tools of a group try's candidate (issue 1921, trycandidates.go): indexer_try_candidates reads the -// candidates the indexer holds for tries (the try, its heads, each member's modules and whether each has its fingerprint), -// and indexer_drop_try_candidate drops one by an ask in takeins. candidates is the candidates state as the tools bundle reads it. -func TryCandidateTools(candidates, recovery Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool { - return []stdio.Tool{ - {Name: TryCandidatesTool, - Description: "The candidates the indexer holds for group tries (a try of several repositories at exact commits, mesh-delivery-checker try): " + - "per try its heads hash, each member's repository and commit, the modules the try touches and, per module, its manifest digest, its " + - "fingerprint or why it has none yet (a merge preview not yet written). The checker's manifest validator and the node-engines' " + - "compose-candidate read this candidate. Read-only; an empty answer means no whole try waits or runs.", - Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { - ctx := context.Background() - keys, err := candidates.Keys(ctx) - if err != nil { - return nil, err - } - var out []meshcatalog.CandidatesByGroupValue - for _, k := range keys { - if !meshcatalogclient.IsTryGroup(k) { - continue - } - v, _, found, err := candidates.Get(ctx, k) - if err != nil || !found { - continue - } - if c, err := DecodeCandidate(v); err == nil { - out = append(out, c) - } - } - // How the recovery of the open tries went: loud when it is not done (since when, why). - said := map[string]any{"done": false, "why": "no recovery has been written yet"} - if v, _, found, err := recovery.Get(ctx, TryRecoveryKey); err == nil && found { - var r TryRecovery - if json.Unmarshal(v, &r) == nil { - said = map[string]any{"done": r.Done} - if !r.Done { - said["why"], said["since"] = r.Why, r.Since - said["says"] = "recovery not done, since " + r.Since.Format(time.RFC3339) + ": " + r.Why - } - } - } - return map[string]any{"tries": out, "recovery": said}, nil - }}, - {Name: RecoverTriesTool, - Description: "Run the recovery of the open tries now: the indexer lists them through the checker's checker_tries, writes the candidate of each " + - "whole try that waits or runs and deletes the candidate of a try no longer listed. It runs by itself at the indexer's start; when it failed " + - "(indexer_try_candidates says recovery not done, since when and why) this runs it again. A hand act: who and why are said in the journal.", - Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{ - "why": map[string]any{"type": "string", "description": "why it is run now, in plain words"}}}, - Run: func(args map[string]any) (any, error) { - why, _ := args["why"].(string) - if strings.TrimSpace(why) == "" { - return nil, fmt.Errorf("say why: a hand act is said in the journal") - } - data, err := json.Marshal(TakeInAsk{V: 1, RecoverTries: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()}) - if err != nil { - return nil, err - } - a, err := ask("tries.recover."+now().UTC().Format("20060102T150405.000000000"), data) - if err != nil { - return nil, err - } - return a.Result, nil - }}, - {Name: DropTryCandidateTool, - Description: "Drop a group try's candidate by hand (the indexer deletes it itself when the try ends): for a candidate left for a try that " + - "is gone, or to make the indexer write it again from the previews at the try's next change. A hand act: who and why are said in the journal.", - Input: map[string]any{"type": "object", "required": []string{"try", "why"}, "properties": map[string]any{ - "try": map[string]any{"type": "string", "description": "the try's id, try-"}, - "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}}, - Run: func(args map[string]any) (any, error) { - id, _ := args["try"].(string) - why, _ := args["why"].(string) - id = strings.TrimSpace(id) - if !meshcatalogclient.IsTryGroup(id) || strings.TrimSpace(why) == "" { - return nil, fmt.Errorf("name the try (try-) and say why") - } - data, err := json.Marshal(TakeInAsk{V: 1, TryCandidate: id, Drop: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()}) - if err != nil { - return nil, err - } - a, err := ask("try-candidate.drop."+id, data) - if err != nil { - return nil, err - } - return a.Result, nil - }}, - } -} diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index 14d0fd53..172f327b 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -77,10 +77,7 @@ func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, d if !ix.PutPreview(key, value, deleted) { return nil } - if err := ix.RecomputeCandidates(ctx); err != nil { - return err - } - return ix.WriteTryCandidates(ctx) // a try that waited for this preview gets its candidate + return ix.RecomputeCandidates(ctx) } // RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index 03766078..8f1b3a5c 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -7,15 +7,11 @@ import ( "fmt" "maps" "os" - "runtime" - "slices" "strings" "testing" - "time" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-checker/checkermsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" @@ -229,221 +225,6 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW if got := fpOf(t, cands); got.Fp2 != "" || got.None2 != meshcatalogclient.NoneNoPreview { t.Errorf("a new head: %+v, want no preview", got) } - // Issue 1921: a whole group try has no pull request, yet the indexer writes its candidate from the checker's try events and - // the previews its fetch wrote (number 0, the heads hash of its members), keeps it apart from the pull requests' candidates, - // and deletes it when the try ends. Its manifests are judged with the seat definitions its members carry, in one locked - // pass, whether the contracts module's manifest changed or only its seat.json did (the wait is bounded: a lock held twice - // would show as a failure here, not as a stuck run). - for _, row := range []struct { - name string - unchanged bool // the contracts module's module.json is the trunk's: only its definitions changed - }{ - {name: "a changed manifest that carries definitions"}, - {name: "an unchanged manifest that carries definitions", unchanged: true}, - } { - t.Run("a group try's candidate: "+row.name, func(t *testing.T) { - ix, mods, cands := previewBed(t, true) - ctx := context.Background() - ix.Verdicts = newMem() - var changed map[string]any - if err := json.Unmarshal([]byte(catalogueManifest(t, "avahi")), &changed); err != nil { - t.Fatal(err) - } - changed["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "go-seats", "kind": "package", "language": "go", "paths": []any{"go"}, "generated-from": "defs"}}} - raw, _ := json.Marshal(changed) - pv := avahiPreview(t, trunkNow, h1) - pm := meshdeliverychecker.PreviewsByRepositoryHeadTrunkValueModulesValue{Tree: "tree-avahi", Manifest: meshcatalogclient.ManifestDigest(raw), Locks: map[string]string{"go.sum": avahiGoSum}} - if row.unchanged { - m, found := moduleOf(t, mods, "avahi") - _ = found - pm.Manifest = m.Manifest - key, _ := meshcatalogclient.ManifestsKey(m.Manifest) - data, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "avahi", Manifest: string(raw)}) - if _, _, _, err := ix.Manifests.Get(ctx, key); err != nil { - t.Fatal(err) - } - _ = ix.Manifests.(*memStore).Update(ctx, key, data, ix.Manifests.(*memStore).revs[key]) - } - pv.Modules["modules/avahi"] = pm - ix.Read = func(_ context.Context, _, _, dir string) (RepoRead, error) { - return RepoRead{Manifests: map[string]string{dir: string(raw)}}, nil - } - listed := 0 - ix.Files = func(_ context.Context, _, commit string, dirs []string) (map[string]map[string]string, error) { - listed++ - if commit != h1 || !slices.Equal(dirs, []string{"modules/avahi/defs/seats"}) { - t.Errorf("listed %v at %s, want the member's head", dirs, commit) - } - return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil - } - ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) { - switch p { - case "modules/avahi/defs/seats/node-engine/seat.json": - return []byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","decision":"d","domain":"Core","verbs":[{"name":"try-verb"}]}`), nil - } - return nil, nil - } - started := `{"id":"try-7"}` - _ = started - members := []TryMember{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}} - held := func() []string { k, _ := cands.Keys(ctx); return k } - if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 0 { - t.Fatalf("a try before its preview: %v, candidates %v", err, held()) - } - done := make(chan struct{}) - go func() { defer close(done); applyPreview(t, ix, pv) }() - select { - case <-done: - case <-time.After(10 * time.Second): - panic("writing the try's candidate hangs: the seat lock is held twice (a locked pass cannot be left by t.Fatal: the lock stays held)") - } - if listed == 0 { - t.Error("the member's definitions were not read") - } - key, _ := meshcatalogclient.CandidatesKey("try-7") - rawc, _, found, _ := cands.Get(ctx, key) - if !found { - t.Fatalf("no candidate for the try after its preview: %v", held()) - } - c, err := DecodeCandidate(rawc) - pull := meshcatalog.CandidatesByGroupValuePullsItem{} - if len(c.Pulls) == 1 { - pull = c.Pulls[0] - } - if row.unchanged { - if err != nil || c.Group != "try-7" || len(pull.Modules) != 0 { - t.Fatalf("candidate = %+v, %v, want the try's group with no module touched", c, err) - } - } else if err != nil || c.Group != "try-7" || pull.Number != 0 || pull.Head != h1 || pull.Fps["avahi"].Manifest == "" { // no fingerprint asserted: this mesh-sdk pin does not read generated-from yet - t.Fatalf("candidate = %+v, %v, want one pull of number 0 with avahi's manifest", c, err) - } - if err := ix.Apply(ctx, forgemsg.Forge{Kind: forgemsg.ForgeOpen}); err != nil || len(held()) != 1 { - t.Errorf("the open message left %v (%v), want the try's candidate alone", held(), err) - } - // The tools: one reads the candidates held for tries, one drops by an ask the indexer answers. - asks := func(key string, value []byte) (TakeInAsk, error) { - a, err := DecodeTakeIn(value) - r := ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why) - a.Result = &r - return a, err - } - recovery := newMem() - ix.TryRecovery = recovery - tools := TryCandidateTools(cands, recovery, asks, time.Now) - if out, err := tools[0].Run(map[string]any{}); err != nil || len(out.(map[string]any)["tries"].([]meshcatalog.CandidatesByGroupValue)) != 1 { - t.Errorf("%s = %v, %v, want the try's candidate", tools[0].Name, out, err) - } - if _, err := tools[2].Run(map[string]any{"try": "feat/x", "why": "w"}); err == nil { - t.Error("a branch was dropped as a try") - } - if out, err := tools[2].Run(map[string]any{"try": "try-7", "why": "by hand"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 { - t.Errorf("drop = %v, %v, candidates %v", out, err, held()) - } - // The start recovered from the checker's tool: an open whole try is written again, one the checker no longer - // lists is deleted, and a try narrowed to a level is none. - ix.OpenTries = func(context.Context) ([]checkermsg.Open, error) { - return []checkermsg.Open{{ID: "try-7", Whole: true, Members: []checkermsg.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}}, - {ID: "try-8", Whole: false, Members: []checkermsg.Member{{Repository: catalogueRepo, Head: h2, Trunk: trunkNow}}}}, nil - } - if err := ix.RecoverTries(ctx); err != nil || len(held()) != 1 { - t.Fatalf("recovered: %v, candidates %v, want try-7's alone", err, held()) - } - ix.OpenTries = func(context.Context) ([]checkermsg.Open, error) { return nil, nil } - if err := ix.RecoverTries(ctx); err != nil || len(held()) != 0 { - t.Errorf("recovered with no open try: %v, candidates %v, want none", err, held()) - } - // A recovery whose call fails is loud and deletes nothing: the candidate stays, indexer_try_candidates says recovery not - // done since when and why, and the next try event runs the recovery again, which clears it; the tool runs it by hand. - if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 { - t.Fatalf("started: %v, candidates %v", err, held()) - } - ix.OpenTries = func(context.Context) ([]checkermsg.Open, error) { - return nil, errors.New("checker_tries does not answer") - } - if err := ix.RecoverTries(ctx); err == nil || len(held()) != 1 { - t.Fatalf("a failed call: %v, candidates %v, want the error and the candidate kept", err, held()) - } - if out, err := tools[0].Run(map[string]any{}); err != nil || !strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "recovery not done, since") || - !strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "checker_tries does not answer") { - t.Errorf("%s = %v, %v, want recovery not done with since and why", tools[0].Name, out, err) - } - ix.OpenTries = func(context.Context) ([]checkermsg.Open, error) { return nil, nil } - if err := ix.StartTry(ctx, "try-11", members); err != nil || len(held()) != 1 || held()[0] != "try-11" { - t.Fatalf("a try event after a failed recovery: %v, candidates %v, want the recovery run (try-7 not listed, gone) and try-11 written", err, held()) - } - if out, _ := tools[0].Run(map[string]any{}); out.(map[string]any)["recovery"].(map[string]any)["done"] != true { - t.Errorf("recovery still said not done after a success: %v", out) - } - ix.OpenTries = func(context.Context) ([]checkermsg.Open, error) { return nil, errors.New("down") } - recoverAsk := func(key string, value []byte) (TakeInAsk, error) { - a, err := DecodeTakeIn(value) - r := ix.RecoverTriesByHand(ctx, a.By, a.Why) - a.Result = &r - return a, err - } - byHand := TryCandidateTools(cands, recovery, recoverAsk, time.Now) - if out, err := byHand[1].Run(map[string]any{"why": "the checker was replaced"}); err != nil || out.(*TakenIn).Stored || !strings.Contains(out.(*TakenIn).Answer, "recovery not done") { - t.Errorf("%s with the checker down = %v, %v", byHand[1].Name, out, err) - } - ix.OpenTries = func(context.Context) ([]checkermsg.Open, error) { return nil, nil } - if out, err := byHand[1].Run(map[string]any{"why": "the checker is back"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 { - t.Errorf("%s = %v, %v, candidates %v, want it recovered (try-11 not listed)", byHand[1].Name, out, err, held()) - } - if err := ix.StartTry(ctx, "try-7", members); err != nil { - t.Fatal(err) - } - // ADR 0638 §3, §4: the consumer exists before the call, so what happens between the call and the hearing is heard after - // it. A try the call listed and whose start is then heard gets one candidate; a try started after the list gets one; a - // try that ended during the call, listed as open, leaves none once its end is heard. - ix.OpenTries = func(context.Context) ([]checkermsg.Open, error) { - return []checkermsg.Open{{ID: "try-7", Whole: true, Members: []checkermsg.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}}, - {ID: "try-9", Whole: true, Members: []checkermsg.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}}}, nil - } - if err := ix.RecoverTries(ctx); err != nil { - t.Fatal(err) - } - for _, step := range []struct { - what string - do func() error - want int - }{ - {"the start heard after the call, of a try the call listed", func() error { return ix.StartTry(ctx, "try-7", members) }, 2}, - {"a start after the list", func() error { return ix.StartTry(ctx, "try-10", members) }, 3}, - {"the end heard of a try ended during the call", func() error { return ix.EndTry(ctx, "try-9") }, 2}, - } { - if err := step.do(); err != nil || len(held()) != step.want { - t.Fatalf("%s: %v, candidates %v, want %d", step.what, err, held(), step.want) - } - } - for _, id := range []string{"try-10"} { - if err := ix.EndTry(ctx, id); err != nil { - t.Fatal(err) - } - } - if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 { - t.Fatalf("started again: %v, candidates %v", err, held()) - } - if err := ix.EndTry(ctx, "try-7"); err != nil || len(held()) != 0 { - t.Errorf("an ended try: %v, candidates %v, want none", err, held()) - } - // The hand recovery never overlaps a try event: a start heard while the call to the checker is out writes its candidate - // before or after the sweep, never between the call and the sweep (which would delete it, the try not being listed). - startDone := make(chan error, 1) - ix.OpenTries = func(context.Context) ([]checkermsg.Open, error) { - go func() { startDone <- ix.StartTry(ctx, "try-12", members) }() - for i := 0; i < 2000 && len(held()) == 0; i++ { - runtime.Gosched() // lets the start write if nothing holds it back - } - return nil, nil - } - if r := ix.RecoverTriesByHand(ctx, "lead", "a test"); !r.Stored { - t.Fatalf("the hand recovery = %+v", r) - } - if err := <-startDone; err != nil || len(held()) != 1 || held()[0] != "try-12" { - t.Errorf("a start heard during the hand recovery: %v, candidates %v, want try-12 kept", err, held()) - } - }) - } // Issue 1921: a head whose tree lacks the directory asked is asked once, however many passes read it, until the head changes; // any other failure (the forge does not answer) is asked again at every pass and said once per head. for _, row := range []struct { diff --git a/modules/mesh-delivery-indexer/internal/index/runcandidates.go b/modules/mesh-delivery-indexer/internal/index/runcandidates.go new file mode 100644 index 00000000..8a66b035 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/runcandidates.go @@ -0,0 +1,296 @@ +package index + +// **A run on request's candidate** (design 47 G §64, §190, §321; A-160, A-174): the runner says `run-started` with the run's +// members, each a module at a named commit on a branch; the indexer builds the run's candidate from it, `candidates.run-`, +// and drops it once every member's pipeline of that run has ended (`pipeline-ended` naming the run). Each member maps to its +// repository and directory by its registration, and its manifest and fingerprint come from the same take-in the indexer +// computes for a watched branch (its files read through the seat git), over the revision view of the member's branch when its +// registration watches that branch, else over the trunk's (A-93). No checker preview is heard and no checker tool is asked. +// +// What the indexer knows of each run is kept in its state runs (key run-: the members and the modules whose pipeline +// ended), written on run-started and on each of the run's pipeline-ended and removed with the candidate, so a restart between +// two members' ends loses nothing (A-174). A pipeline-ended heard before its run-started is kept all the same: the run-started +// that follows finds it. + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// RunsState is the indexer's state of the runs on request it holds a candidate for (A-174): the bucket +// mesh-delivery-indexer_runs, one key per run, run-. +const ( + RunsState = "runs" + RunsBucket = Module + "_" + RunsState +) + +// Run is one value of the state runs: the run's members as run-started named them, and the modules whose pipeline of the +// run has ended. +type Run struct { + V int `json:"v"` + Run string `json:"run"` + Members []meshpipelinerunner.Member `json:"members,omitempty"` + Ended []string `json:"ended,omitempty"` + At time.Time `json:"at"` +} + +// RunGroup is the candidate's group, and the runs key, of a run: run- (A-160). +func RunGroup(run string) (string, error) { + g := "run-" + strings.TrimSpace(run) + if strings.TrimSpace(run) == "" || !meshcatalogclient.IsTryGroup(g) { + return "", fmt.Errorf("%w: %q is no run's id", ErrUnreadable, run) + } + return g, nil +} + +// done says every member's pipeline of the run has ended; a run whose run-started is not heard yet is not done. +func (r Run) done() bool { + if len(r.Members) == 0 { + return false + } + for _, m := range r.Members { + if !slices.Contains(r.Ended, m.Module) { + return false + } + } + return true +} + +// StartRun takes a run on request in, as the runner said it started: its members are kept in the state runs and its +// candidate is written; a run every member of which has ended already (its ends heard first) is dropped instead. +func (ix *Indexer) StartRun(ctx context.Context, e meshpipelinerunner.RunStarted) error { + group, err := RunGroup(e.Run) + if err != nil { + return err + } + if len(e.Members) == 0 { + return fmt.Errorf("%w: the run %s names no member", ErrUnreadable, e.Run) + } + ix.runMu.Lock() + defer ix.runMu.Unlock() + r, err := ix.updateRun(ctx, group, func(r *Run) { r.Members = slices.Clone(e.Members) }) + if err != nil { + return err + } + if r.done() { + return ix.dropRun(ctx, group) + } + return ix.writeRunCandidate(ctx, group, r.Members) +} + +// EndRunPipeline takes one pipeline-ended in: of a run's pipeline, its module is kept as ended, and once every member's has +// ended the candidate and the run's key are removed. A pipeline of no run is none of this. +func (ix *Indexer) EndRunPipeline(ctx context.Context, e meshpipelinerunner.PipelineEnded) error { + if e.Run == "" { + return nil + } + group, err := RunGroup(e.Run) + if err != nil { + return err + } + ix.runMu.Lock() + defer ix.runMu.Unlock() + r, err := ix.updateRun(ctx, group, func(r *Run) { + if e.Module != "" && !slices.Contains(r.Ended, e.Module) { + r.Ended = append(r.Ended, e.Module) + slices.Sort(r.Ended) + } + }) + if err != nil { + return err + } + if !r.done() { + return nil + } + return ix.dropRun(ctx, group) +} + +// updateRun reads the run's key, changes it and writes it back, made when absent. +func (ix *Indexer) updateRun(ctx context.Context, group string, change func(*Run)) (Run, error) { + if ix.Runs == nil { + return Run{}, fmt.Errorf("the state %s is not bound, so the run %s is not kept", RunsState, group) + } + var out Run + err := write(ctx, ix.Runs, group, func(value []byte, found bool) ([]byte, bool, error) { + r := Run{V: 1, Run: strings.TrimPrefix(group, "run-")} + if found { + if err := json.Unmarshal(value, &r); err != nil { + return nil, false, fmt.Errorf("%s.%s holds what is not a run: %w", RunsState, group, err) + } + } + change(&r) + r.At = ix.now().UTC() + out = r + data, err := json.Marshal(r) + return data, false, err + }) + return out, err +} + +// dropRun removes the run's candidate, then its key. +func (ix *Indexer) dropRun(ctx context.Context, group string) error { + if ix.Store != nil { + key, err := meshcatalogclient.CandidatesKey(group) + if err != nil { + return err + } + if err := write(ctx, ix.Store, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil { + return fmt.Errorf("the candidate of %s is not dropped: %w", group, err) + } + } + if err := write(ctx, ix.Runs, group, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }); err != nil { + return fmt.Errorf("the run %s is not removed: %w", group, err) + } + ix.logf("the run %s: every member's pipeline ended, its candidate is dropped", group) + return nil +} + +// RunBase is the branch whose revision view a member's take-in starts from (A-174): the member's own branch when its +// registration watches it, else the trunk. +func RunBase(reg meshcatalog.Registration, branch string) string { + if branch != "" && slices.Contains(reg.Branches, branch) { + return branch + } + return TrunkBranch +} + +// runMember is one member placed: its registration's repository and directory, and the branch its take-in starts from. +type runMember struct { + meshpipelinerunner.Member + repository, dir, base string +} + +// writeRunCandidate writes the run's candidate: one pull of number 0 per member, its module's manifest and fingerprint as +// the take-in of its commit over its base's revision view computes them, its manifests judged with the seat definitions +// the members carry, as a revision's are. +func (ix *Indexer) writeRunCandidate(ctx context.Context, group string, members []meshpipelinerunner.Member) error { + if ix.Store == nil || ix.Read == nil { + return nil + } + if ix.Registry == nil || ix.Registry.Registrations == nil || ix.Revisions == nil { + return fmt.Errorf("the run %s: the indexer holds no registrations or no revisions, so its members cannot be taken in", group) + } + var placed []runMember + for _, m := range members { + reg, found, err := ix.Registry.Registration(ctx, m.Module) + if err != nil { + return fmt.Errorf("the run %s: the registration of %s: %w", group, m.Module, err) + } + if !found { + return fmt.Errorf("%w: the run %s: %s has no registration, so its repository is not known", ErrUnreadable, group, m.Module) + } + dir := strings.Trim(reg.Path, "/") + if dir == meshcatalogclient.RootPath { + dir = "" + } + placed = append(placed, runMember{Member: m, repository: trunk.RepositoryName(reg.Origin), dir: dir, base: RunBase(reg, m.Branch)}) + } + slices.SortFunc(placed, func(a, b runMember) int { + return strings.Compare(a.repository+"\x00"+a.Module, b.repository+"\x00"+b.Module) + }) + heads := make([]meshdeliveryclient.Head, len(placed)) + for i, m := range placed { + heads[i] = meshdeliveryclient.Head{Repository: m.repository, Head: m.Commit} + } + hash, err := meshdeliveryclient.HeadsHash(heads) + if err != nil { + return err + } + pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(placed)) + var entries []PendingEntry + over := map[string]string{} + // One take-in per base branch: the members on one base are taken in together, so one member sees another's. + bases := map[string]*dryRun{} + for i, m := range placed { + d, ok := bases[m.base] + if !ok { + view, nodes, _, err := BranchView(ctx, ix.Revisions, m.base) + if err != nil { + return fmt.Errorf("the run %s: the revision view of %s: %w", group, m.base, err) + } + d = ix.dry() + d.ix.Modules, d.ix.Graph, d.ix.Taken = view, nodes, newOverlay(nil) + d.ix.Logf = func(format string, args ...any) { + ix.logf("the run "+group+", computed and not written: "+format, args...) + } + bases[m.base] = d + } + read, err := ix.Read(ctx, m.repository, m.Commit, m.dir) + if err != nil { + return fmt.Errorf("the run %s: reading %s of %s at %.12s: %w", group, m.Module, m.repository, m.Commit, err) + } + raw := read.Manifests[m.dir] + if raw == "" { + return fmt.Errorf("%w: the run %s: %s at %.12s holds no manifest at %q", ErrUnreadable, group, m.repository, m.Commit, m.dir) + } + if name := manifestModule(raw); name != m.Module { + return fmt.Errorf("%w: the run %s: the manifest at %q of %s at %.12s names %q, not %s", ErrUnreadable, group, m.dir, m.repository, m.Commit, name, m.Module) + } + f := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeMerged, Repository: m.repository, Merge: m.Commit, Root: read.Root, + Trees: map[string]string{m.dir: read.Trees[m.dir]}, Manifests: map[string]string{m.dir: raw}} + if l := read.Locks[m.dir]; len(l) > 0 { + f.Locks = map[string]map[string]string{m.dir: l} + } + if p := read.Packages[m.dir]; len(p) > 0 { + f.Packages = map[string][]string{m.dir: p} + } + if _, err := d.ix.applyManifests(ctx, f, false); err != nil { + return fmt.Errorf("the run %s: taking %s in: %w", group, m.Module, err) + } + key, err := meshcatalogclient.ModulesKey(m.Module) + if err != nil { + return err + } + value, _, found, err := d.ix.Modules.Get(ctx, key) + if err != nil { + return err + } + if !found { + return fmt.Errorf("the run %s: the take-in of %s at %.12s wrote no module (the journal says why)", group, m.Module, m.Commit) + } + mv, err := meshcatalogclient.DecodeModule(value) + if err != nil { + return err + } + pulls[i] = meshcatalog.CandidatesByGroupValuePullsItem{Repository: m.repository, Head: m.Commit, Modules: []string{m.Module}, + Fps: map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{m.Module: {Manifest: mv.Manifest, Fp2: mv.Fp2, None2: mv.None2, + Packages: mv.Packages}}} + over[m.Module] = mv.Manifest + entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.repository, Merge: m.Commit, Manifests: map[string]string{m.dir: raw}}}) + } + seats, err := ix.revisionSeats(ctx, entries) + if err != nil { + return fmt.Errorf("the run %s: reading the seat definitions of its members: %w", group, err) + } + ix.judgeDigestsWith(ctx, seats, over, "the run "+group) + c := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: group, Heads: hash, Pulls: pulls} + c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls) + if err := ix.refuseDisagreement(c); err != nil { + return err + } + data, err := EncodeCandidate(c) + if err != nil { + return err + } + key, err := meshcatalogclient.CandidatesKey(group) + if err != nil { + return err + } + if err := write(ctx, ix.Store, key, func(_ []byte, _ bool) ([]byte, bool, error) { return data, false, nil }); err != nil { + return err + } + ix.logf("the run %s: its candidate of %d members is written", group, len(placed)) + return nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go b/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go new file mode 100644 index 00000000..dfd17632 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go @@ -0,0 +1,130 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" +) + +// Design 47 G §64, §190, A-160, A-174: a run on request's candidate is built from the runner's run-started, each member taken +// in from its registration's repository at its commit (read through the seat git), and dropped once every member's pipeline of +// the run has ended; what the indexer knows of the run is kept in its state runs, so a restart between two ends loses nothing. +func TestARunsCandidateIsBuiltOnItsStartAndDroppedWhenEveryMembersPipelineEnded(t *testing.T) { + const c1, c2 = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222" + bed := func(t *testing.T) (*Indexer, *memStore, *memStore, *int) { + t.Helper() + ix, _, _ := manifestsIndexer(t) + cands, runs := newMem(), newMem() + ix.Store, ix.Runs, ix.Revisions = cands, runs, newMemObjects(0) + ix.Registry = &Registry{Registrations: MemoryStore()} + for _, r := range []meshcatalog.Registration{ + {Module: "avahi", Origin: catalogueRepo, Path: "modules/avahi", Branches: []string{"main"}}, + {Module: "netcheck", Origin: "novox/mesh-networking", Path: "modules/netcheck", Branches: []string{"main", "beta"}}, + } { + data, _ := json.Marshal(r) + if err := ix.Registry.Registrations.Create(context.Background(), RegistrationKey(r.Module), data); err != nil { + t.Fatal(err) + } + } + reads := 0 + ix.Read = func(_ context.Context, repository, commit, dir string) (RepoRead, error) { + reads++ + module := map[string]string{"modules/avahi": "avahi", "modules/netcheck": "netcheck"}[dir] + return RepoRead{Root: "r", Trees: map[string]string{dir: "tree-" + commit[:4]}, Manifests: map[string]string{dir: catalogueManifest(t, module)}, + Locks: map[string]map[string]string{dir: {"go.sum": avahiGoSum}}}, nil + } + return ix, cands, runs, &reads + } + ctx := context.Background() + started := meshpipelinerunner.RunStarted{V: 1, Run: "7", By: "lead@shanks", Members: []meshpipelinerunner.Member{ + {Module: "avahi", Commit: c1, Branch: "feature/1966-x"}, {Module: "netcheck", Commit: c2, Branch: "beta"}}} + ended := func(module string) meshpipelinerunner.PipelineEnded { + return meshpipelinerunner.PipelineEnded{V: 1, Pipeline: "p", Module: module, Branch: "b", Commit: "c", Outcome: "passed", Run: "7"} + } + key, _ := meshcatalogclient.CandidatesKey("run-7") + t.Run("built on its start, one pull per member, and dropped with the run after a restart", func(t *testing.T) { + ix, cands, runs, reads := bed(t) + if err := ix.StartRun(ctx, started); err != nil { + t.Fatal(err) + } + raw, _, found, _ := cands.Get(ctx, key) + c, err := DecodeCandidate(raw) + if !found || err != nil || c.Group != "run-7" || len(c.Pulls) != 2 || *reads != 2 { + t.Fatalf("candidate %+v (%v, found %v, %d reads), want run-7 with two pulls read through the seat git", c, err, found, *reads) + } + for _, p := range c.Pulls { + m := p.Modules[0] + if p.Number != 0 || len(p.Modules) != 1 || p.Fps[m].Manifest == "" || (p.Fps[m].Fp2 == "" && p.Fps[m].None2 == "") || + (m == "avahi") != (p.Repository == catalogueRepo && p.Head == c1) { + t.Errorf("pull %+v, want number 0, its registration's repository at its commit, a manifest and a fingerprint or why none", p) + } + } + if err := ix.EndRunPipeline(ctx, ended("avahi")); err != nil { + t.Fatal(err) + } + // A restart: a fresh indexer over the same states. + again := &Indexer{Store: cands, Runs: runs, Revisions: ix.Revisions, Registry: ix.Registry, Read: ix.Read} + if _, _, found, _ := cands.Get(ctx, key); !found { + t.Fatal("the candidate went before its last member's pipeline ended") + } + if err := again.EndRunPipeline(ctx, ended("netcheck")); err != nil { + t.Fatal(err) + } + if _, _, found, _ := cands.Get(ctx, key); found { + t.Error("the candidate stays after every member's pipeline ended") + } + if k, _ := runs.Keys(ctx); len(k) != 0 { + t.Errorf("runs holds %v after the run's end, want none", k) + } + }) + t.Run("ends heard before the start drop it at the start", func(t *testing.T) { + ix, cands, runs, _ := bed(t) + for _, m := range []string{"avahi", "netcheck"} { + if err := ix.EndRunPipeline(ctx, ended(m)); err != nil { + t.Fatal(err) + } + } + if err := ix.StartRun(ctx, started); err != nil { + t.Fatal(err) + } + k1, _ := cands.Keys(ctx) + k2, _ := runs.Keys(ctx) + if len(k1)+len(k2) != 0 { + t.Errorf("candidates %v, runs %v, want none", k1, k2) + } + }) + t.Run("a member with no registration is never taken in; a pipeline of no run is none of this", func(t *testing.T) { + ix, cands, _, _ := bed(t) + lost := started + lost.Members = []meshpipelinerunner.Member{{Module: "gone", Commit: c1}} + if err := ix.StartRun(ctx, lost); !errors.Is(err, ErrUnreadable) { + t.Errorf("a member with no registration: %v, want unreadable", err) + } + if err := ix.EndRunPipeline(ctx, meshpipelinerunner.PipelineEnded{Module: "avahi"}); err != nil { + t.Error(err) + } + if k, _ := cands.Keys(ctx); len(k) != 0 { + t.Errorf("candidates %v, want none", k) + } + }) + t.Run("a member's take-in starts from its own branch only when its registration watches it", func(t *testing.T) { + reg := meshcatalog.Registration{Branches: []string{"main", "beta"}} + for branch, want := range map[string]string{"beta": "beta", "main": TrunkBranch, "feature/1966-x": TrunkBranch, "": TrunkBranch} { + if got := RunBase(reg, branch); got != want { + t.Errorf("RunBase(%q) = %q, want %q", branch, got, want) + } + } + }) + t.Run("run- is a run's group beside try-, never a pull request's", func(t *testing.T) { + for g, want := range map[string]bool{"run-7": true, "run-a1-b": true, "try-3": true, "run-": false, "run-a.b": false, "feat/x": false} { + if meshcatalogclient.IsTryGroup(g) != want { + t.Errorf("IsTryGroup(%q) = %v, want %v", g, !want, want) + } + } + }) +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index f007fddb..74aa1eb2 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -57,13 +57,9 @@ type TakeInAsk struct { Merge string `json:"merge,omitempty"` MergeRepository string `json:"merge-repository,omitempty"` Sequence uint64 `json:"sequence,omitempty"` - // TryCandidate asks the candidate of this try's group to be dropped (indexer_drop_try_candidate), trycandidates.go. - TryCandidate string `json:"try-candidate,omitempty"` // Fingerprints asks the fingerprints of Module ("all": every module) to be computed again now, from the published // packages as they are read (indexer_fingerprints_again, fingerprints.go). Fingerprints bool `json:"fingerprints,omitempty"` - // RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries). - RecoverTries bool `json:"recover-tries,omitempty"` // Recompute asks the catalog graph to be computed again now and compared with the newest kept revision (indexer_recompute). Recompute bool `json:"recompute,omitempty"` // CandidateGroup and CandidateHeads, with Drop, ask that candidate's catalog half to be dropped (indexer_drop_candidate_graph). @@ -95,8 +91,8 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.Registry == nil { - return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try") + if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.Registry == nil { + return a, errors.New("a takeins value naming no module, repository, pending wait or merge") } return a, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/trycandidates.go b/modules/mesh-delivery-indexer/internal/index/trycandidates.go deleted file mode 100644 index 9c00c6d0..00000000 --- a/modules/mesh-delivery-indexer/internal/index/trycandidates.go +++ /dev/null @@ -1,437 +0,0 @@ -package index - -// **A group try's candidate** (issue 1921; novox/hq ADR 0593 §7): a try of several repositories at exact commits has no pull -// request, so the forge bridge sends the indexer no message for it; yet the node-engine's compose-candidate and the checker's -// manifest validator read the indexer's candidate of a group. The indexer therefore watches the checker's state tries (the -// checker keeps it; this module reads it) and, for a try that is not narrowed to a level or to some tests, writes -// `candidates.`: one pull per member (number 0), its modules and manifests taken from the merge preview the try's -// fetch wrote, its manifests kept and judged with the seat definitions the members carry, as a take-in judges a revision's. -// The fingerprints come with the previews (RecomputeCandidates). The candidate is deleted when the try ends. `try-` -// is the reserved group name of a try: the bridge's open message never takes it out, and the checker never checks it. - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "path" - "slices" - "strings" - "time" - - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdeliverychecker" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-sdk/go/keys" -) - -// TryCandidateTools: the tools that read, drop and recover a try's candidate. -const ( - TryCandidatesTool = "indexer_try_candidates" - DropTryCandidateTool = "indexer_drop_try_candidate" - RecoverTriesTool = "indexer_recover_tries" - // TryRecoveryState is the indexer's state of how the last recovery of the open tries went: the bucket - // mesh-delivery-indexer_try-recovery, one key (TryRecoveryKey). - TryRecoveryState = "try-recovery" - TryRecoveryBucket = Module + "_" + TryRecoveryState - TryRecoveryKey = "state" -) - -// TryRecovery is the value of the try-recovery state: whether the candidates of the open tries were recovered at the -// indexer's start or by hand, and when it began to fail, why. -type TryRecovery struct { - V int `json:"v"` - Done bool `json:"done"` - Since time.Time `json:"since,omitzero"` - Why string `json:"why,omitempty"` - At time.Time `json:"at"` -} - -// TryMember is one repository of a try: its exact commit and the trunk commit it is merged onto. -type TryMember struct{ Repository, Head, Trunk string } - -// StartTry takes a whole try in, as the checker said it started (checkermsg.Started): its members at their commits are kept, and -// its candidate is written once every member's preview is held. The checker's state is never read (ADR 0596). -func (ix *Indexer) StartTry(ctx context.Context, id string, members []TryMember) error { - ix.tryMu.Lock() - defer ix.tryMu.Unlock() - ix.retryRecovery(ctx) - return ix.startTry(ctx, id, members) -} - -// retryRecovery runs the recovery again on a try event when the last one failed: the failure stays said until one succeeds. -func (ix *Indexer) retryRecovery(ctx context.Context) { - if ix.recoveryFailed.Load() && ix.OpenTries != nil { - _ = ix.recoverNow(ctx) - } -} - -func (ix *Indexer) startTry(ctx context.Context, id string, members []TryMember) error { - if !meshcatalogclient.IsTryGroup(id) || len(members) == 0 { - return fmt.Errorf("%q with %d members is no try to start", id, len(members)) - } - ix.mu.Lock() - if ix.tries == nil { - ix.tries = map[string][]TryMember{} - } - ix.tries[id] = slices.Clone(members) - ix.mu.Unlock() - return ix.writeTryCandidate(ctx, id) -} - -// EndTry takes a try out, as the checker said it ended (checkermsg.Ended), and deletes its candidate. -func (ix *Indexer) EndTry(ctx context.Context, id string) error { - ix.tryMu.Lock() - defer ix.tryMu.Unlock() - ix.retryRecovery(ctx) - return ix.endTry(ctx, id) -} - -func (ix *Indexer) endTry(ctx context.Context, id string) error { - ix.mu.Lock() - delete(ix.tries, id) - ix.mu.Unlock() - if !meshcatalogclient.IsTryGroup(id) { - return nil - } - return ix.dropTryCandidate(ctx, id) -} - -// RecoverTries is, at the indexer's start, what the events it missed would have said: the candidates of the tries that wait -// or run now are written, and the candidate of a try the checker no longer lists is deleted. The checker's tool lists them -// (OpenTries); its bucket is not read. Nothing to recover from when no part is wired to ask. -// -// A recovery that fails is said loudly and kept (the try-recovery state, read by indexer_try_candidates) until one succeeds: -// the events are still heard after it, and indexer_recover_tries runs it again by hand. -// -// A recovery and a try event never overlap (tryMu): a sweep that deleted the candidate a start event wrote after the call to the -// checker would leave a running try with none. -func (ix *Indexer) RecoverTries(ctx context.Context) error { - ix.tryMu.Lock() - defer ix.tryMu.Unlock() - return ix.recoverNow(ctx) -} - -func (ix *Indexer) recoverNow(ctx context.Context) error { - if ix.OpenTries == nil || ix.Store == nil { - return nil - } - err := ix.recoverTries(ctx) - ix.recoveryFailed.Store(err != nil) - ix.noteRecovery(ctx, err) - return err -} - -// noteRecovery writes how a recovery went: failing keeps the time it began to fail, and the journal says it each time. -func (ix *Indexer) noteRecovery(ctx context.Context, err error) { - now := ix.now() - r := TryRecovery{V: 1, Done: err == nil, At: now.UTC()} - if err != nil { - r.Why, r.Since = err.Error(), now.UTC() - ix.logf("RECOVERY OF THE TRIES NOT DONE: %v (the try candidates may be stale or missing until indexer_recover_tries or the next start succeeds)", err) - } - if ix.TryRecovery == nil { - return - } - for range writeTries { - cur, rev, found, gerr := ix.TryRecovery.Get(ctx, TryRecoveryKey) - if gerr != nil { - ix.logf("the try-recovery state cannot be read: %v", gerr) - return - } - if err != nil && found { - var was TryRecovery - if json.Unmarshal(cur, &was) == nil && !was.Done && !was.Since.IsZero() { - r.Since = was.Since - } - } - data, _ := json.Marshal(r) - var werr error - if found { - werr = ix.TryRecovery.Update(ctx, TryRecoveryKey, data, rev) - } else { - werr = ix.TryRecovery.Create(ctx, TryRecoveryKey, data) - } - if werr == nil { - return - } - if !errors.Is(werr, ErrConflict) { - ix.logf("the try-recovery state cannot be written: %v", werr) - return - } - } -} - -// RecoverTriesByHand runs the recovery now (indexer_recover_tries), a hand act said in the journal. -func (ix *Indexer) RecoverTriesByHand(ctx context.Context, by, why string) TakenIn { - r := TakenIn{Module: "tries"} - ix.logf("recovery of the tries asked by %s (%s)", by, why) - if ix.OpenTries == nil { - r.Answer = "the indexer is wired to list no open try: nothing to recover from" - return r - } - if err := ix.RecoverTries(ctx); err != nil { - r.Answer = "recovery not done: " + err.Error() - return r - } - r.Stored, r.Answer = true, "the candidates of the open tries are recovered" - return r -} - -func (ix *Indexer) recoverTries(ctx context.Context) error { - open, err := ix.OpenTries(ctx) - if err != nil { - return fmt.Errorf("the open tries cannot be listed, so a missed try event is not recovered: %w", err) - } - live := map[string]bool{} - for _, o := range open { - if !o.Whole || len(o.Members) == 0 { - continue - } - live[o.ID] = true - var ms []TryMember - for _, m := range o.Members { - ms = append(ms, TryMember{m.Repository, m.Head, m.Trunk}) - } - if err := ix.startTry(ctx, o.ID, ms); err != nil { - return err - } - } - keys, err := ix.Store.Keys(ctx) - if err != nil { - return err - } - for _, k := range keys { - if meshcatalogclient.IsTryGroup(k) && !live[k] { - if err := ix.endTry(ctx, k); err != nil { - return err - } - } - } - return nil -} - -// WriteTryCandidates writes the candidate of every live try that has none yet and whose previews are all there: after a -// preview was written, the try that waited for it. -func (ix *Indexer) WriteTryCandidates(ctx context.Context) error { - ix.mu.Lock() - ids := make([]string, 0, len(ix.tries)) - for id := range ix.tries { - ids = append(ids, id) - } - ix.mu.Unlock() - slices.Sort(ids) - for _, id := range ids { - if err := ix.writeTryCandidate(ctx, id); err != nil { - return err - } - } - return nil -} - -func (ix *Indexer) dropTryCandidate(ctx context.Context, id string) error { - if ix.Store == nil { - return nil - } - key, err := meshcatalogclient.CandidatesKey(id) - if err != nil { - return err - } - return write(ctx, ix.Store, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }) -} - -// DropTryCandidate deletes a try's candidate by hand (indexer_drop_try_candidate); a live try gets it again at its next -// change. -func (ix *Indexer) DropTryCandidate(ctx context.Context, id, by, why string) TakenIn { - r := TakenIn{Module: id} - switch { - case !meshcatalogclient.IsTryGroup(id): - r.Answer = id + " is no try's group, try-: nothing dropped" - default: - if err := ix.dropTryCandidate(ctx, id); err != nil { - r.Answer = "the candidate of " + id + " could not be dropped: " + err.Error() - } else { - r.Stored, r.Answer = true, "the candidate of "+id+" is dropped by "+by+" ("+why+")" - ix.logf("%s", r.Answer) - } - } - return r -} - -// writeTryCandidate writes the candidate of a live try when it has none at its members' heads and every member's merge -// preview is held. -func (ix *Indexer) writeTryCandidate(ctx context.Context, id string) error { - if ix.Store == nil || ix.Read == nil { - return nil - } - ix.mu.Lock() - members := slices.Clone(ix.tries[id]) - ix.mu.Unlock() - if len(members) == 0 { - return nil - } - key, err := meshcatalogclient.CandidatesKey(id) - if err != nil { - return err - } - slices.SortFunc(members, func(a, b TryMember) int { return strings.Compare(a.Repository, b.Repository) }) - heads := make([]meshdeliveryclient.Head, len(members)) - for i, m := range members { - heads[i] = meshdeliveryclient.Head{Repository: m.Repository, Head: m.Head} - } - hash, err := meshdeliveryclient.HeadsHash(heads) - if err != nil { - return err - } - if value, _, found, err := ix.Store.Get(ctx, key); err == nil && found { - if c, err := DecodeCandidate(value); err == nil && c.Heads == hash { - return nil // fingerprints follow the previews (RecomputeCandidates) - } - } - previews := make([]meshdeliverychecker.PreviewsByRepositoryHeadTrunkValue, len(members)) - for i, m := range members { - pk, err := PreviewKey(m.Repository, m.Head, m.Trunk) - if err != nil { - return fmt.Errorf("the try %s: %w", id, err) - } - ix.mu.Lock() - pv, ok := ix.previews[pk] - ix.mu.Unlock() - if !ok { - return nil // its fetch has not written the preview yet: this is called again when it does - } - previews[i] = pv - } - held, err := ix.heldModules(ctx) - if err != nil { - return err - } - pulls := make([]meshcatalog.CandidatesByGroupValuePullsItem, len(members)) - var entries []PendingEntry - over := map[string]string{} - for i, m := range members { - pull := meshcatalog.CandidatesByGroupValuePullsItem{Repository: heads[i].Repository, Head: heads[i].Head, Modules: []string{}} - // defs are every module directory's manifest of the member at its head, the unchanged ones too: a contracts module whose - // seat.json changed while its module.json did not carries definitions all the same (ADR 0636 §4). - defs := map[string]string{} - for _, dir := range slices.Sorted(mapKeys(previews[i].Modules)) { - pm := previews[i].Modules[dir] - name, unchanged := "", false - for n, h := range held { - if strings.EqualFold(h.Repository, m.Repository) && h.Path == forgeDir(dir) { - name, unchanged = n, h.Manifest == pm.Manifest - break - } - } - if unchanged { - if raw := ix.keptManifest(ctx, pm.Manifest); raw != "" { - defs[dir] = raw - } - continue // the manifest the trunk holds: the try does not touch the module - } - rr, err := ix.Read(ctx, m.Repository, m.Head, forgeDir(dir)) - if err != nil { - return fmt.Errorf("the try %s: reading the manifest of %s at %.12s: %w", id, dir, m.Head, err) - } - raw := rr.Manifests[forgeDir(dir)] - if raw == "" { - ix.logf("the try %s: %s holds no manifest at %.12s, so no module of it is taken", id, dir, m.Head) - continue - } - module := manifestModule(raw) - if module == "" { - module = name - } - if module == "" { - module = path.Base(strings.Trim(dir, "/.")) - } - digest, err := ix.keepManifest(ctx, module, raw) - if err != nil { - return err - } - if digest != pm.Manifest { - ix.logf("the try %s: the manifest of %s read at %.12s is not the one its preview names, left", id, dir, m.Head) - continue - } - if pull.Fps == nil { - pull.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{} - } - pull.Modules = append(pull.Modules, module) - pull.Fps[module] = meshcatalog.CandidatesByGroupValuePullsItemFpsValue{Manifest: digest, None2: meshcatalogclient.NoneNoPreview} - defs[dir] = raw - over[module] = digest - } - pulls[i] = pull - entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.Repository, Merge: m.Head, Manifests: defs}}) - } - // The manifests are judged with the seat definitions the members carry (seatsof.go, ADR 0636 §4), in one pass that takes the - // seats. Their verdicts are the verdicts state's, keyed by the manifest's digest, as a pull request's candidate's are - // (judgeDigests in ApplyMessage): nothing reads them but the engines' composition of that digest and the module - // value of a trunk module with the same digest. - seats, err := ix.revisionSeats(ctx, entries) - if err != nil { - return fmt.Errorf("the try %s: reading the seat definitions of its members: %w", id, err) - } - ix.judgeDigestsWith(ctx, seats, over, "the try "+id) - c := meshcatalog.CandidatesByGroupValue{V: keys.Version, Group: id, Heads: hash, Pulls: pulls} - c.Fpset = meshcatalogclient.FingerprintSet(c.Pulls) - ix.withPreviews(ctx, &c) - if err := ix.refuseDisagreement(c); err != nil { - return err - } - data, err := EncodeCandidate(c) - if err != nil { - return err - } - return write(ctx, ix.Store, key, func(_ []byte, _ bool) ([]byte, bool, error) { return data, false, nil }) -} - -// keptManifest is the text of the manifest the indexer keeps under a digest, or empty. -func (ix *Indexer) keptManifest(ctx context.Context, digest string) string { - if ix.Manifests == nil || digest == "" { - return "" - } - key, err := meshcatalogclient.ManifestsKey(digest) - if err != nil { - return "" - } - value, _, found, err := ix.Manifests.Get(ctx, key) - if err != nil || !found { - return "" - } - m, err := meshcatalogclient.DecodeManifest(value) - if err != nil { - return "" - } - return m.Manifest -} - -func mapKeys[V any](m map[string]V) func(func(string) bool) { - return func(yield func(string) bool) { - for k := range m { - if !yield(k) { - return - } - } - } -} - -// forgeDir is a preview's directory as the forge reader spells it: the root as "". -func forgeDir(dir string) string { - if dir == "." || dir == "/" { - return "" - } - return strings.Trim(dir, "/") -} - -// heldAt says the trunk holds a module of the repository in that directory. -func heldAt(held map[string]meshcatalog.ModulesByModuleValue, repository, dir string) bool { - for _, h := range held { - if strings.EqualFold(h.Repository, repository) && h.Path == forgeDir(dir) { - return true - } - } - return false -} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 56b7eeda..6ea9b9cc 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -126,7 +126,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("uses = %v, invokes = %v, want the seat operator-channel used and seat:mesh-module-assigner.assignments invoked", m.Uses, m.Invokes) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: index.RunsState, History: 1}, {Name: "own-build", History: 1}, {Name: index.PackagesState, History: index.PackagesHistory}, {Name: views.MarksState, History: 1}, {Name: index.FindingsState, History: 5}, {Name: views.StatusState, History: 1}, {Name: index.RequirementsState, History: 5}, {Name: index.SeatDefinitionsState, History: 5}, {Name: index.PendingState, History: 1}, {Name: index.UnjudgedState, History: 1}, {Name: index.EntryIndexState, History: 1, MaxMiB: 8}, {Name: index.NamedState, History: 1, MaxMiB: 4}, // design 47 G §695 @@ -166,8 +166,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // internal/broker/state.go:62-93). // The registry seats' published fill each (§413: no consume of version.numbered) package's series (DLV-I5, design 51 *The versions*). if want := []string{"go-package-registry.published", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", - "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", - "mesh-delivery.source-moved", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + "mesh-delivery-checker.verdict", "mesh-delivery-forge-bridge.forge.**", + "mesh-delivery.source-moved", "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.run-started", "mesh-wants.version-delivered", "mesh-wants.wait-ended", "mesh-wants.wanted", "node-build-go.step-ended", "node-engine.running", "npm-package-registry.published", "operator-channel.decided"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -179,7 +179,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"mesh-delivery-assigner.assignments", "seat:git.trees", "seat:git.file", "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", "seat:mesh-module-assigner.assignments", "seat:mesh-module-assigner.placement", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.steps", "seat:node-build-node.steps", "seat:node-build-image.steps", "mesh-delivery-builder.builder_versions"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle @@ -198,12 +198,12 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build } want = append(want, "indexer_first_views", "indexer_first_view_again", index.ConfigReadersTool, index.BusStateTool, index.RequirementsTool, index.PackageUsersTool, index.SayAcceptedTool, // DLV-F3r, DLV-I10, DLV-I11, DLV-I7, DLV-I8, DLV-I9c index.SeatDefinitionTool, index.SeatDefinitionsAgainTool, index.OutOfStepTool, // DLV-I19 - index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5) + index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TakeInMergeTool, index.FingerprintsAgainTool, // issue 1966; 1585/T12, T13 (ADR 0633 §5) // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool) // design 47 G §667, §8 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 47 || slices.Contains(m.Tools, "modules") { + if !reflect.DeepEqual(m.Tools, want) || len(want) != 44 || slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 4f6e4770..30b23dc5 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -58,9 +58,6 @@ "indexer_pending", "indexer_take_in_pending", "indexer_drop_pending", - "indexer_try_candidates", - "indexer_drop_try_candidate", - "indexer_recover_tries", "indexer_take_in_merge", "indexer_fingerprints_again", "indexer_catalog_revision", @@ -81,9 +78,6 @@ "mesh-build-node.toolchain.reported", "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.assignments", - "mesh-delivery-checker.preview.written", - "mesh-delivery-checker.try.ended", - "mesh-delivery-checker.try.started", "mesh-builds.package-published", "mesh-builds.toolchain-agreed", "mesh-wants.version-delivered", @@ -93,6 +87,8 @@ "mesh-delivery-forge-bridge.forge.**", "mesh-delivery.source-moved", "mesh-wants.wanted", + "mesh-pipeline-runner.run-started", + "mesh-pipeline-runner.pipeline-ended", "node-engine.running", "operator-channel.decided", "npm-package-registry.published" @@ -107,7 +103,7 @@ { "name": "takeins", "history": 1 }, { "name": "verdicts", "history": 5 }, { "name": "oldsections", "history": 1 }, - { "name": "try-recovery", "history": 1 }, + { "name": "runs", "history": 1 }, { "name": "own-build", "history": 1 }, { "name": "packages", "history": 5 }, { "name": "view-marks", "history": 1 }, @@ -160,7 +156,6 @@ "seat:git.contains", "seat:git.open-pulls", "seat:git.branch", - "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition", "seat:mesh-delivery.source", From 5b13cc7ab1e809c29dfad057d2d59099da62894b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:53:45 +0100 Subject: [PATCH 06/24] IX24: the indexer's four kept calls are asked again only while their ask is open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit messageVia, stepsVia, sourceVia and versionsVia go through CallKeptWhileOpen: open while the commit's take-in is pending, the candidate is held, the branch is watched, the module is in the newest view. A call that ends with its ask closed is index.ErrAskClosed, not an unanswered call. Builds: design 47 G §351, §367, §460, §531; A-190 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../mesh-delivery-indexer/bridgemessage.go | 10 +- .../cmd/mesh-delivery-indexer/keptopen.go | 29 +++++ .../mesh-delivery-indexer/keptopen_test.go | 38 ++++++ .../cmd/mesh-delivery-indexer/main.go | 9 +- .../cmd/mesh-delivery-indexer/main_test.go | 5 +- .../cmd/mesh-delivery-indexer/source.go | 12 +- .../cmd/mesh-delivery-indexer/steps.go | 9 +- .../cmd/mesh-delivery-indexer/versions.go | 9 +- .../internal/index/open_asks.go | 104 ++++++++++++++++ .../internal/index/open_asks_test.go | 111 ++++++++++++++++++ .../internal/index/pending.go | 2 +- .../internal/index/seed_steps.go | 2 + .../internal/index/source.go | 2 +- .../internal/index/versions_fill.go | 7 +- .../internal/index/versions_fill_edges.go | 2 +- .../internal/index/versions_test.go | 4 +- 16 files changed, 327 insertions(+), 28 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/keptopen.go create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/keptopen_test.go create mode 100644 modules/mesh-delivery-indexer/internal/index/open_asks.go create mode 100644 modules/mesh-delivery-indexer/internal/index/open_asks_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go index f99bb36a..e56ae60d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/bridgemessage.go @@ -14,14 +14,16 @@ import ( const mergedTool = index.ForgeBridgeModule + ".bridge_merged_message" // messageVia asks the forge bridge for a merge's merged message, for a held merge the indexer does not hold in memory (a -// restart, a new build). An error is the bridge's own words. -func messageVia(call toolCaller) func(ctx context.Context, repository, merge string, number int) ([]byte, error) { - return func(_ context.Context, repository, merge string, number int) ([]byte, error) { +// restart, a new build), a kept call asked again only while the commit's take-in is pending (A-190). An error is the +// bridge's own words. +func messageVia(call openCaller, pending func(ctx context.Context, repository, merge string) (bool, error), logf func(string, ...any)) func(ctx context.Context, repository, merge string, number int) ([]byte, error) { + return func(ctx context.Context, repository, merge string, number int) ([]byte, error) { ask := map[string]any{"repository": repository, "merge": merge} if number > 0 { ask["number"] = number } - raw, err := call(mergedTool, ask) + raw, err := keptWhileOpen(ctx, call, logf, "message."+repository+"."+merge, mergedTool, ask, + func(ctx context.Context) (bool, error) { return pending(ctx, repository, merge) }) if err != nil { return nil, err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/keptopen.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/keptopen.go new file mode 100644 index 00000000..866cab6a --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/keptopen.go @@ -0,0 +1,29 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/busretry" + + "mesh-delivery-indexer/internal/index" +) + +// openCaller is process.Conn.CallKeptWhileOpen: a kept call asked again only while its ask is open (design 47 G §351, +// §460, §531; A-190). +type openCaller func(o busretry.Open, key string, body any) (json.RawMessage, error) + +// keptWhileOpen asks verb as a kept call keyed by key, asked again only while open says what the ask serves still stands. +// A call that ends with its ask closed is index.ErrAskClosed, never an unanswered call (A-190). +func keptWhileOpen(ctx context.Context, call openCaller, logf func(string, ...any), key, verb string, body any, + open func(context.Context) (bool, error)) (json.RawMessage, error) { + raw, err := call(busretry.Open{Key: key, Open: func(string) (bool, error) { return open(ctx) }, Logf: logf}, verb, body) + if err == nil { + return raw, nil + } + if still, oerr := open(ctx); oerr == nil && !still { + return nil, fmt.Errorf("%s: %w", verb, index.ErrAskClosed) + } + return nil, err +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/keptopen_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/keptopen_test.go new file mode 100644 index 00000000..a6742daa --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/keptopen_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "git.novox.be/novox/mesh-sdk/go/busretry" + + "mesh-delivery-indexer/internal/index" +) + +// A-190: the four asks pass their key's open function to the kept call; a call that fails with its key closed is +// index.ErrAskClosed, one that fails while it is open keeps its own error. +func TestKeptWhileOpenNamesAClosedAsk(t *testing.T) { + ctx := context.Background() + failing := errors.New("no responders") + for _, c := range []struct { + name string + open bool + answer error + want error + }{{"open", true, failing, failing}, {"closed", false, failing, index.ErrAskClosed}, {"answered", false, nil, nil}} { + var asked busretry.Open + call := func(o busretry.Open, key string, _ any) (json.RawMessage, error) { + asked = o + return json.RawMessage(`{}`), c.answer + } + _, err := keptWhileOpen(ctx, call, t.Logf, "k1", "tool", nil, func(context.Context) (bool, error) { return c.open, nil }) + if !errors.Is(err, c.want) { + t.Errorf("%s: err = %v, want %v", c.name, err, c.want) + } + if got, _ := asked.Open("k1"); asked.Key != "k1" || got != c.open { + t.Errorf("%s: the kept call was given key %q open %v", c.name, asked.Key, got) + } + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 8136c950..62b17d88 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -449,7 +449,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ReadOpen: openVia(calls.Call), Packages: awaitedStore{b: packages, js: js, name: index.PackagesBucket}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), OwnBuildState: awaitedStore{b: ownBuildState}, Heard: heardStore, Placement: placementVia(calls.CallKept), RunningWhole: runningWholeVia(js), TryRecovery: awaitedStore{b: tryRecovery}, Graph: graph, SeatDefinitions: awaitedStore{b: seatDefs}, ReadFile: readFileVia(calls.Call), SaySeatDefinition: saySeats, - Message: messageVia(calls.CallKept), Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) + Revisions: revisions, EntryIndex: awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.EntryIndexBucket, "no entry is found by its fingerprint", logger.Printf)}, // Message a kept call: a read, called again through a transport fault (ADR 0668 §1) // Each catalog revision may bring another holder of a build seat: the build modules are found again (buildmodules.go). SayCatalogRevision: afterRevision(refreshingBuildModules(sayCatalogRevision(conn, cred), &buildFollow), &fillFollow), SayFill: sayFill(conn, cred), @@ -469,11 +469,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.RegistrationsBucket, "no registration is kept or written", logger.Printf)}, awaitedStore{b: process.AwaitBucket(ctx, js, conn, index.ProposalsBucket, "no change of a registration waits for its warrant", logger.Printf)}, routerRecordOf(cred.Module, calls.Call, calls.Ask), ix, logger.Printf) + ix.Message = messageVia(calls.CallKeptWhileOpen, ix.TakeInPending, logger.Printf) // a kept call, called again through a transport fault while the commit's take-in is pending (ADR 0668 §1, A-190) registry.Pruned = ix.PruneBranchBuilds ix.Registry = registry named.Changed = ix.RemoveUnlisted // The first fill of the versions (design 47 G §372): at the start, and again at each catalog revision kept until it is made. - fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKept), logf: logger.Printf} + fill := &versionsFill{ix: ix, ctx: ctx, ask: versionsVia(calls.CallKeptWhileOpen, ix.ModuleInNewestView, logger.Printf), logf: logger.Printf} fillFollow.Store(fill) fill.kick() ix.SeedModules = func(ctx context.Context) ([]index.SeedModule, error) { @@ -481,7 +482,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if r == nil { return nil, errors.New("the seat list is not read yet: the seed waits for it") } - return ix.SeedModulesFrom(ctx, r.Definitions(), stepsVia(calls.CallKept)) + return ix.SeedModulesFrom(ctx, r.Definitions(), stepsVia(calls.CallKeptWhileOpen, ix.SeedEntryHeld, logger.Printf)) } // Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the // node-engine names the successor while it replaces this process, and an old process that learnt it later would take @@ -518,7 +519,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The source revision of a watched branch (design 47 G §15, §115, §362): asked of the seat mesh-delivery's verb source as a // kept call, once at the start, before the handler of its event source-moved reads anything; a source-moved heard // meanwhile is read after. - source := &sourceFollow{read: sourceVia(calls.CallKept), apply: ix.ApplySource, applyBranch: ix.ApplyBranchSource, logf: logger.Printf} + source := &sourceFollow{read: sourceVia(ctx, calls.CallKeptWhileOpen, ix.BranchWatched, logger.Printf), apply: ix.ApplySource, applyBranch: ix.ApplyBranchSource, logf: logger.Printf} go source.catchUp(ctx) // Only the toolchain reports of nodes the catalogue places the build module on count (issue 1940, ADR 0549 §3): the // assignments is settled first, so the toolchains settle through the gate already knowing it. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go index eea5e715..75144627 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/busretry" "log" "reflect" "strings" @@ -234,7 +235,7 @@ func TestTheSourceIsReadFromTheSeatAndASourceMovedDuringTheCatchUpIsReadAfterIt( func TestTheSeatsSourceAnswerIsReadAndItsErrorIsAnError(t *testing.T) { // design 47 G §115 var asked any - read := sourceVia(func(key string, body any) (json.RawMessage, error) { + read := sourceVia(context.Background(), func(_ busretry.Open, key string, body any) (json.RawMessage, error) { asked = body if key != "seat:mesh-delivery.source" { t.Errorf("asked %q", key) @@ -243,7 +244,7 @@ func TestTheSeatsSourceAnswerIsReadAndItsErrorIsAnError(t *testing.T) { // desig return json.RawMessage(`{"error":"source.bad is not readable"}`), nil } return json.RawMessage(`{"branch":"main","revision":7,"repositories":[{"repository":"novox/mesh-delivery","head":"` + head + `","trunk":"main"}]}`), nil - }) + }, func(context.Context, string) (bool, error) { return true, nil }, t.Logf) a, err := read("main") if err != nil || a.Revision != 7 || len(a.Repositories) != 1 || asked.(meshdelivery.SourceInput).Branch != "main" { t.Errorf("source = %+v, %v", a, err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go index b7c6e267..c9bc6cc2 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/source.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "maps" "slices" @@ -12,17 +13,19 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-sdk/go/keys" "github.com/nats-io/nats.go/jetstream" + "mesh-delivery-indexer/internal/index" ) // sourceInvoke is the seat mesh-delivery's verb source as the manifest's invokes names it: a watched branch's source // revision, asked of the seat and never read from the gate's state (design 47 G §15, §115; ADR 0596 §2). const sourceInvoke = "seat:" + meshdelivery.Seat + "." + meshdelivery.VerbSource -// sourceVia asks the seat for one branch's source revision through call (process.Conn.CallKept, a read) and reads its +// sourceVia asks the seat for one branch's source revision through call (process.Conn.CallKeptWhileOpen, a read asked again while the branch is watched, A-190) and reads its // declared answer; an answer that carries an error is that error. -func sourceVia(call toolCaller) func(branch string) (meshdelivery.SourceAnswer, error) { +func sourceVia(ctx context.Context, call openCaller, watched func(ctx context.Context, branch string) (bool, error), logf func(string, ...any)) func(branch string) (meshdelivery.SourceAnswer, error) { return func(branch string) (meshdelivery.SourceAnswer, error) { - raw, err := call(sourceInvoke, meshdelivery.SourceInput{Branch: branch}) + raw, err := keptWhileOpen(ctx, call, logf, "source."+branch, sourceInvoke, meshdelivery.SourceInput{Branch: branch}, + func(ctx context.Context) (bool, error) { return watched(ctx, branch) }) if err != nil { return meshdelivery.SourceAnswer{}, err } @@ -97,6 +100,9 @@ func (s *sourceFollow) moved(ctx context.Context, branch string) { // through the take-in, another watched branch's into its own revision (A-93). func (s *sourceFollow) take(ctx context.Context, branch string) { a, err := s.read(branch) + if errors.Is(err, index.ErrAskClosed) { // the branch is no longer watched: not said unanswered (A-190) + return + } if err != nil { s.logf("source.%s is not read, and is read again at its next source-moved: %v", branch, err) return diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/steps.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/steps.go index 1dea5455..0a3cfb56 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/steps.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/steps.go @@ -10,12 +10,13 @@ import ( "mesh-delivery-indexer/internal/index" ) -// stepsVia asks a build seat's verb steps through call (process.Conn.CallKept, ADR 0668 §1) and reads its answer (design +// stepsVia asks a build seat's verb steps through call (process.Conn.CallKeptWhileOpen, ADR 0668 §1, asked again while the candidate is held, A-190) and reads its answer (design // 47 G §704); the seats' verbs have one form, so nodebuildgo's contract reads each seat's answer. -func stepsVia(call toolCaller) index.StepsCaller { - return func(_ context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) { +func stepsVia(call openCaller, held func(ctx context.Context, module, fingerprint string) (bool, error), logf func(string, ...any)) index.StepsCaller { + return func(ctx context.Context, seat string, in nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) { verb := "seat:" + seat + "." + index.StepsVerb - raw, err := call(verb, in) + raw, err := keptWhileOpen(ctx, call, logf, "steps."+seat+"."+in.Module+"."+in.Fingerprint, verb, in, + func(ctx context.Context) (bool, error) { return held(ctx, in.Module, in.Fingerprint) }) if err != nil { return nodebuildgo.StepsAnswer{}, err } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go index 3e750328..cb5dc56d 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/versions.go @@ -136,11 +136,12 @@ func sayVersionsFilled(conn *nats.Conn, cred process.Account) func(context.Conte } } -// versionsVia asks the builder's tool builder_versions for one name as a kept call (ADR 0668 §1, §2) and reads its answer. -func versionsVia(call toolCaller) index.VersionsAsker { - return func(_ context.Context, name string) (index.BuilderVersionsAnswer, error) { +// versionsVia asks the builder's tool builder_versions for one name as a kept call (ADR 0668 §1, §2) asked again while the module is in the newest view (A-190) and reads its answer. +func versionsVia(call openCaller, inView func(ctx context.Context, module string) (bool, error), logf func(string, ...any)) index.VersionsAsker { + return func(ctx context.Context, module, name string) (index.BuilderVersionsAnswer, error) { var a index.BuilderVersionsAnswer - raw, err := call(index.BuilderVersionsCall, map[string]any{"name": name}) + raw, err := keptWhileOpen(ctx, call, logf, "versions."+name, index.BuilderVersionsCall, map[string]any{"name": name}, + func(ctx context.Context) (bool, error) { return inView(ctx, module) }) if err != nil { return a, err } diff --git a/modules/mesh-delivery-indexer/internal/index/open_asks.go b/modules/mesh-delivery-indexer/internal/index/open_asks.go new file mode 100644 index 00000000..576dc802 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/open_asks.go @@ -0,0 +1,104 @@ +package index + +import ( + "context" + "errors" + "fmt" + "slices" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// ErrAskClosed is what a kept call answers when its ask was closed while it was made again (design 47 G §351, §460, A-190): +// what the ask served no longer stands, so the call stopped and is not said unanswered. +var ErrAskClosed = errors.New("the ask is closed: what it served no longer stands") + +// takingIn marks a context as the take-in of one commit's merged message, which is pending while that context lives. +type takingIn struct{ repository, merge string } + +type takingInKey struct{} + +// forTakeIn is ctx marked as the take-in of the merged message of repository's commit merge. +func forTakeIn(ctx context.Context, repository, merge string) context.Context { + return context.WithValue(ctx, takingInKey{}, takingIn{repository, merge}) +} + +// TakeInPending says the take-in of the commit merge of repository is still pending (A-190, for the forge bridge's +// merged_message): its take-in is the one ctx is made for, or a pending entry names it. +func (ix *Indexer) TakeInPending(ctx context.Context, repository, merge string) (bool, error) { + if ctx.Err() != nil { + return false, nil + } + if t, ok := ctx.Value(takingInKey{}).(takingIn); ok && t.repository == repository && t.merge == merge { + return true, nil + } + if ix.Pending == nil { + return false, nil + } + entries, err := ReadPending(ctx, ix.Pending, ix.logf) + if err != nil { + return false, err + } + return slices.ContainsFunc(entries, func(e PendingEntry) bool { + return e.Forge.Merge == merge && e.Forge.Repository == repository + }), nil +} + +// SeedEntryHeld says the newest trunk revision still holds the entry of module at the fingerprint fp, the candidate the +// seed asks steps for (A-190). +func (ix *Indexer) SeedEntryHeld(ctx context.Context, module, fp string) (bool, error) { + if ctx.Err() != nil || ix.Revisions == nil { + return false, ctx.Err() + } + rev, found, err := newestTrunk(ctx, ix.Revisions) + if err != nil || !found { + return false, err + } + _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) + if err != nil { + return false, err + } + return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module && e.Fp == fp }), nil +} + +// ModuleInNewestView says module is among the entries of the newest trunk revision, the view the versions are filled for (A-190). +func (ix *Indexer) ModuleInNewestView(ctx context.Context, module string) (bool, error) { + if ctx.Err() != nil || ix.Revisions == nil { + return false, ctx.Err() + } + rev, found, err := newestTrunk(ctx, ix.Revisions) + if err != nil || !found { + return false, err + } + _, entries, err := ReadRevision(ctx, ix.Revisions, fmt.Sprintf("trunk.%d", rev.Revision)) + if err != nil { + return false, err + } + return slices.ContainsFunc(entries, func(e meshcatalog.CatalogEntry) bool { return e.Module == module }), nil +} + +// BranchWatched says branch (as the source's key names it, escaped; empty asks the list of every branch) is watched: the +// trunk always, another branch while a registration names it (A-190). With no registry, every branch is. +func (ix *Indexer) BranchWatched(ctx context.Context, branch string) (bool, error) { + if ctx.Err() != nil { + return false, nil + } + if isTrunkBranch(branch) || ix.Registry == nil || ix.Registry.Registrations == nil { + return true, nil + } + name, err := keys.UnescapeName(branch) + if err != nil { + return false, nil + } + regs, err := ix.Registry.RegistrationsAnswer(ctx, meshcatalog.RegistrationsInput{}) + if err != nil { + return false, err + } + for _, r := range regs.Registrations { + if slices.Contains(r.Branches, name) { + return true, nil + } + } + return false, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/open_asks_test.go b/modules/mesh-delivery-indexer/internal/index/open_asks_test.go new file mode 100644 index 00000000..7471afeb --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/open_asks_test.go @@ -0,0 +1,111 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// A-190: a kept call is asked again only while what it serves still stands; each of the indexer's four has its predicate. +func TestKeptCallsOpenWhileTheirAskServesSomething(t *testing.T) { + ctx := context.Background() + t.Run("the take-in of a commit is pending while an entry names it or the take-in is made", func(t *testing.T) { + pending := newMem() + val, _ := encodePending(Pending{Seq: 1, Ref: &PendingRef{Repository: "novox/r", Merge: "c1"}}) + if err := pending.Create(ctx, "m.1", val); err != nil { + t.Fatal(err) + } + ix := &Indexer{Pending: pending} + cancelled, cancel := context.WithCancel(ctx) + cancel() + for _, c := range []struct { + name string + ctx context.Context + repo, commit string + want bool + }{ + {"held", ctx, "novox/r", "c1", true}, + {"taken in or dropped", ctx, "novox/r", "c2", false}, + {"in hand", forTakeIn(ctx, "novox/r", "c2"), "novox/r", "c2", true}, + {"in hand for another commit", forTakeIn(ctx, "novox/r", "c3"), "novox/r", "c2", false}, + {"the take-in's context ended", cancelled, "novox/r", "c1", false}, + } { + if got, err := ix.TakeInPending(c.ctx, c.repo, c.commit); err != nil || got != c.want { + t.Errorf("%s: pending = %v, %v, want %v", c.name, got, err, c.want) + } + } + }) + t.Run("the seed's candidate and the versions' module are in the newest trunk revision", func(t *testing.T) { + ix := &Indexer{Revisions: newMemObjects(1 << 20)} + keep := func(n int64, fp string) { + rev, objects, err := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: n, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Fp: fp, Manifest: map[string]any{}}}) + if err == nil { + err = KeepRevision(ctx, ix.Revisions, rev, objects) + } + if err != nil { + t.Fatal(err) + } + } + keep(1, "fp1") + if held, err := ix.SeedEntryHeld(ctx, "avahi", "fp1"); err != nil || !held { + t.Errorf("the entry at its fingerprint: %v, %v", held, err) + } + if in, err := ix.ModuleInNewestView(ctx, "avahi"); err != nil || !in { + t.Errorf("the module in the view: %v, %v", in, err) + } + if in, _ := ix.ModuleInNewestView(ctx, "gone"); in { + t.Error("a module the view does not hold is in it") + } + keep(2, "fp2") + if held, _ := ix.SeedEntryHeld(ctx, "avahi", "fp1"); held { + t.Error("a candidate the newest revision replaced is still held") + } + }) + t.Run("a branch is watched while a registration names it", func(t *testing.T) { + g := newRegRig() + ix := &Indexer{Registry: g.r} + raw, _ := json.Marshal(meshcatalog.Registration{Module: "avahi", Branches: []string{"main", "feat"}}) + if err := g.r.Registrations.Create(ctx, "avahi", raw); err != nil { + t.Fatal(err) + } + if w, err := ix.BranchWatched(ctx, TrunkBranch); err != nil || !w { + t.Errorf("the trunk: %v, %v", w, err) + } + if w, _ := ix.BranchWatched(ctx, "feat"); !w { + t.Error("a branch a registration names is not watched") + } + if w, _ := ix.BranchWatched(ctx, "other"); w { + t.Error("a branch no registration names is watched") + } + }) +} + +// A-190: a call that ends with its ask closed is not said unanswered: no finding, and the fill and the seed go on. +func TestAClosedAskIsNotUnanswered(t *testing.T) { + ctx := context.Background() + closed := fmt.Errorf("steps: %w", ErrAskClosed) + ix := &Indexer{Findings: newFindings(), Heard: newMem(), Revisions: newMemObjects(1 << 20), Versions: newMem()} + rev, objects, _ := CatalogRevisionOf(meshcatalog.CatalogRevision{Revision: 1, Kind: "trunk"}, map[string]meshcatalog.CatalogEntry{"avahi": {Module: "avahi", Fp: "fp", Manifest: map[string]any{}, + Artifacts: []meshcatalog.Artifact{{Ref: "avahi:program", Language: "go"}}}}) + if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { + t.Fatal(err) + } + defs := map[string]seatdef.Definition{"node-build-go": {Acts: []seatdef.ActingRule{{Step: seatdef.StepBuild, On: "go"}}}} + if _, err := ix.SeedModulesFrom(ctx, defs, func(context.Context, string, nodebuildgo.StepsInput) (nodebuildgo.StepsAnswer, error) { + return nodebuildgo.StepsAnswer{}, closed + }); err != nil || len(ix.stepsSince) != 0 { + t.Errorf("a closed steps ask counted unanswered: %v %v", err, ix.stepsSince) + } + if err := ix.FillVersions(ctx, func(context.Context, string, string) (BuilderVersionsAnswer, error) { + return BuilderVersionsAnswer{}, fmt.Errorf("builder_versions: %w", ErrAskClosed) + }); errors.Is(err, ErrAskClosed) || (err != nil && strings.Contains(err.Error(), "is not answered")) { + t.Errorf("a closed versions ask is unanswered: %v", err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 3005971b..22a18610 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -1490,7 +1490,7 @@ func (ix *Indexer) TakeInMerge(ctx context.Context, repository, merge string, se r.Answer = "this indexer does not ask the forge bridge" return r } - data, err := ix.Message(ctx, repository, merge, 0) + data, err := ix.Message(forTakeIn(ctx, repository, merge), repository, merge, 0) if err != nil { r.Answer = fmt.Sprintf("the forge bridge could not answer for the merge of %s at %s: %v", repository, merge, err) return r diff --git a/modules/mesh-delivery-indexer/internal/index/seed_steps.go b/modules/mesh-delivery-indexer/internal/index/seed_steps.go index c945beb8..06b69a88 100644 --- a/modules/mesh-delivery-indexer/internal/index/seed_steps.go +++ b/modules/mesh-delivery-indexer/internal/index/seed_steps.go @@ -2,6 +2,7 @@ package index import ( "context" + "errors" "fmt" "maps" "path" @@ -91,6 +92,7 @@ func (ix *Indexer) SeedModulesFrom(ctx context.Context, defs map[string]seatdef. Fetch: &nodebuildgo.Fetch{Repository: e.Repository, Commit: e.Commit, Dir: e.Path, Paths: namedPaths(e, name), Packages: maps.Clone(e.Packages)}} ans, err := steps(ctx, seat, in) switch { + case errors.Is(err, ErrAskClosed): // the entry is no longer held: not said unanswered (A-190) case err != nil: unanswered[seat] = err case ans.Error != "": diff --git a/modules/mesh-delivery-indexer/internal/index/source.go b/modules/mesh-delivery-indexer/internal/index/source.go index 739ff2e0..19e75efd 100644 --- a/modules/mesh-delivery-indexer/internal/index/source.go +++ b/modules/mesh-delivery-indexer/internal/index/source.go @@ -216,7 +216,7 @@ func (ix *Indexer) CatchUp(ctx context.Context) TakenIn { // mergedAt is the forge bridge's merged message of a repository's head. func (ix *Indexer) mergedAt(ctx context.Context, repository, head string) (forgemsg.Forge, error) { - data, err := ix.Message(ctx, repository, head, 0) + data, err := ix.Message(forTakeIn(ctx, repository, head), repository, head, 0) if err != nil { return forgemsg.Forge{}, fmt.Errorf("the forge bridge could not answer: %w", err) } diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill.go b/modules/mesh-delivery-indexer/internal/index/versions_fill.go index 9f520236..e2b960b5 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_fill.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill.go @@ -46,7 +46,7 @@ type BuilderVersionsAnswer struct { } // VersionsAsker asks builder_versions for a name's versions (a kept call, ADR 0668 §1, §2). -type VersionsAsker func(ctx context.Context, name string) (BuilderVersionsAnswer, error) +type VersionsAsker func(ctx context.Context, module, name string) (BuilderVersionsAnswer, error) // versionNameOf is the name the builder numbers an artifact under: its package's name in its ecosystem, else // / (images, bundles and archives). @@ -97,7 +97,10 @@ func (ix *Indexer) FillVersions(ctx context.Context, ask VersionsAsker) error { per := []found1{} for _, a := range e.Artifacts { name := versionNameOf(e.Module, a) - ans, err := ask(ctx, name) + ans, err := ask(ctx, e.Module, name) + if errors.Is(err, ErrAskClosed) { + continue // the module left the view: nothing waits on its versions (A-190) + } if err != nil { unanswered = err continue diff --git a/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go b/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go index 7a2268e9..9a77267d 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_fill_edges.go @@ -58,7 +58,7 @@ func (ix *Indexer) filledEdges(ctx context.Context, entry meshcatalog.CatalogEnt } ans, ok := answers[name] if !ok { - a, err := ask(ctx, name) + a, err := ask(ctx, module, name) if err != nil { continue } diff --git a/modules/mesh-delivery-indexer/internal/index/versions_test.go b/modules/mesh-delivery-indexer/internal/index/versions_test.go index 9acc013b..f81918b5 100644 --- a/modules/mesh-delivery-indexer/internal/index/versions_test.go +++ b/modules/mesh-delivery-indexer/internal/index/versions_test.go @@ -115,7 +115,7 @@ func TestTheFirstFillOfTheVersionsComesFromTheBuildersPublished(t *testing.T) { t.Fatal(err) } up := false - ask := func(_ context.Context, name string) (BuilderVersionsAnswer, error) { + ask := func(_ context.Context, _, name string) (BuilderVersionsAnswer, error) { if !up { return BuilderVersionsAnswer{}, errors.New("no responders") } @@ -457,7 +457,7 @@ func TestTheFirstFillGivesTheNewestVersionsTheirFilledEdges(t *testing.T) { if err := KeepRevision(ctx, ix.Revisions, rev, objects); err != nil { t.Fatal(err) } - ask := func(_ context.Context, name string) (BuilderVersionsAnswer, error) { + ask := func(_ context.Context, _, name string) (BuilderVersionsAnswer, error) { pub := func(v, d string, at time.Time) BuilderVersion { return BuilderVersion{Version: v, State: "published", Digest: d, Revision: "9", At: at} } From c2950e332fda43ae313c92e2b14b432be338bba9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:57:11 +0100 Subject: [PATCH 07/24] A-194: indexer_runs reads the state runs and the run candidates, indexer_run_drop drops one by hand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The drop is a hand act recorded through mesh-hand-acts before it is asked; the indexer, the state's owner, drops the run's candidate and its key. Builds: design 47 G §64; A-194 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-delivery-indexer-tools/main.go | 4 + .../cmd/mesh-delivery-indexer/takein.go | 4 + .../internal/index/runcandidates_test.go | 58 ++++++ .../internal/index/runs_tools.go | 179 ++++++++++++++++++ .../internal/index/takein.go | 4 +- .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 4 +- 7 files changed, 253 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/runs_tools.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 4a13173e..1509c9c2 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -304,6 +304,10 @@ func main() { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.UnpinTool, Verb: "unpin"}) return err })...) + tools = append(tools, index.RunsTools(keptRead{stdio.State(index.RunsState)}, keptRead{stdio.State(index.CandidatesState)}, asks, time.Now, func(what, why string) error { + _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: index.RunDropTool, Verb: "run-drop"}) + return err + })...) tools = append(tools, index.ConsumerTools(keptRead{stdio.State(process.ConsumersState)})...) tools = append(tools, index.RegistrationTools(keptRead{stdio.State(index.ProposalsState)}, asks, time.Now, func(cause, verb, what, why string) error { _, err := acts.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause, Verb: verb}) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index f876df51..e72897a0 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -27,6 +27,7 @@ type takerIn interface { DropCandidateGraph(ctx context.Context, group, heads, by, why string) index.TakenIn NamedDrop(ctx context.Context, fingerprint, member, by, why string) index.TakenIn Unpin(ctx context.Context, revision, record, by, why string) index.TakenIn + DropRunByHand(ctx context.Context, run, by, why string) index.TakenIn RegistryAct(ctx context.Context, a index.TakeInAsk) index.TakenIn } @@ -60,6 +61,9 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream } else if a.UnpinRevision != "" { asked, r = "unpin of a record from a catalog graph revision", ix.Unpin(ctx, a.UnpinRevision, a.UnpinRecord, a.By, a.Why) a.Module = r.Module + } else if a.RunDrop != "" { + asked, r = "drop of a run on request", ix.DropRunByHand(ctx, a.RunDrop, a.By, a.Why) + a.Module = r.Module } else if a.Fingerprints { asked, r = "computing of fingerprints again", ix.FingerprintsAgain(ctx, a.Module) } else if a.Merge != "" { diff --git a/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go b/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go index dfd17632..2f6c4cfa 100644 --- a/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go +++ b/modules/mesh-delivery-indexer/internal/index/runcandidates_test.go @@ -4,7 +4,9 @@ import ( "context" "encoding/json" "errors" + "slices" "testing" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -128,3 +130,59 @@ func TestARunsCandidateIsBuiltOnItsStartAndDroppedWhenEveryMembersPipelineEnded( } }) } + +// Design 47 G §64, A-194: the state runs and the run candidates are read by indexer_runs and a run is dropped by hand by +// indexer_run_drop, its hand act written first. +func TestTheRunsAreReadAndDroppedByHand(t *testing.T) { + ctx := context.Background() + cands, runs := newMem(), newMem() + ix := &Indexer{Store: cands, Runs: runs, Now: func() time.Time { return time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC) }} + put := func(s *memStore, key string, v any) { + t.Helper() + data, _ := json.Marshal(v) + if err := s.Create(ctx, key, data); err != nil { + t.Fatal(err) + } + } + const id = "0a1b2c3d4e5f" + put(runs, "run-"+id, Run{V: 1, Run: id, Members: []meshpipelinerunner.Member{{Module: "avahi", Commit: "c1"}, {Module: "netcheck", Commit: "c2"}}, Ended: []string{"avahi"}}) + for _, g := range []string{"run-" + id, "run-ffffffffffff"} { + key, _ := meshcatalogclient.CandidatesKey(g) + put(cands, key, map[string]string{"group": g}) + } + read, err := ReadRuns(ctx, runs, cands) + if err != nil || len(read.Runs) != 1 || read.Runs[0].Run != id || !read.Runs[0].Candidate || !slices.Equal(read.Runs[0].Waiting, []string{"netcheck"}) || !slices.Equal(read.Orphans, []string{"run-ffffffffffff"}) { + t.Fatalf("runs = %+v, %v, want the run waiting for netcheck with its candidate and one orphan candidate", read, err) + } + var recorded []string + var asked []string + tools := RunsTools(runs, cands, func(key string, value []byte) (TakeInAsk, error) { + asked = append(asked, key) + a, err := DecodeTakeIn(value) + if err != nil { + return a, err + } + a.Result = &TakenIn{} + *a.Result = ix.DropRunByHand(ctx, a.RunDrop, a.By, a.Why) + return a, nil + }, ix.now, func(what, why string) error { recorded = append(recorded, what); return nil }) + if len(tools) != 2 || tools[0].Name != RunsTool || tools[1].Name != RunDropTool { + t.Fatalf("tools = %v", tools) + } + if _, err := tools[1].Run(map[string]any{"run": id}); err == nil || len(recorded) != 0 { + t.Fatalf("a drop with no why was done: %v, %v", err, recorded) + } + if _, err := tools[1].Run(map[string]any{"run": id, "why": "its pipelines never end"}); err != nil || len(recorded) != 1 || len(asked) != 1 { + t.Fatalf("drop: %v, recorded %v, asked %v: want the hand act recorded, then the ask", err, recorded, asked) + } + read, _ = ReadRuns(ctx, runs, cands) + if len(read.Runs) != 0 || len(read.Orphans) != 1 { + t.Errorf("after the drop: %+v, want the run and its candidate gone and the orphan left", read) + } + if got := ix.DropRunByHand(ctx, "ffffffffffff", "lead", "an orphan"); !got.Stored { + t.Errorf("an orphan candidate is not dropped: %+v", got) + } + if got := ix.DropRunByHand(ctx, id, "lead", "again"); got.Stored { + t.Errorf("a run already dropped was dropped again: %+v", got) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/runs_tools.go b/modules/mesh-delivery-indexer/internal/index/runs_tools.go new file mode 100644 index 00000000..7135bc3c --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/runs_tools.go @@ -0,0 +1,179 @@ +package index + +// The tools of the state runs and the run candidates (design 47 G §64; A-174, A-194): indexer_runs reads every run the indexer +// holds a candidate for, and indexer_run_drop drops one by hand, its hand act written first. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +const ( + // RunsTool reads the runs; RunDropTool drops one by hand. + RunsTool = "indexer_runs" + RunDropTool = "indexer_run_drop" +) + +// RunRead is one run of the state runs as indexer_runs lists it. +type RunRead struct { + // Run is the run's id as the runner says it, Group the candidate's group and the key of the state (run-). + Run string `json:"run"` + Group string `json:"group"` + Members []meshpipelinerunner.Member `json:"members,omitempty"` + // Ended are the modules whose pipeline of the run has ended, Waiting the members whose has not. + Ended []string `json:"ended,omitempty"` + Waiting []string `json:"waiting,omitempty"` + At time.Time `json:"at"` + // Candidate says the run's candidate is in the state candidates. + Candidate bool `json:"candidate"` +} + +// RunsRead is the answer of indexer_runs: the runs the state holds, and the run candidates no run of the state holds. +type RunsRead struct { + Runs []RunRead `json:"runs"` + // Orphans are the groups of the candidates that are a run's (run-) while the state runs holds no such run. + Orphans []string `json:"orphans,omitempty"` +} + +// ReadRuns reads the state runs and the run candidates of the state candidates; neither is written. +func ReadRuns(ctx context.Context, runs, candidates Store) (RunsRead, error) { + out := RunsRead{Runs: []RunRead{}} + if runs == nil { + return out, fmt.Errorf("the state %s is not bound", RunsState) + } + held := map[string]bool{} // the candidates' keys + if candidates != nil { + ks, err := candidates.Keys(ctx) + if err != nil { + return out, fmt.Errorf("listing %s: %w", CandidatesState, err) + } + for _, k := range ks { + held[k] = true + } + } + names, err := runs.Keys(ctx) + if err != nil { + return out, fmt.Errorf("listing %s: %w", RunsState, err) + } + slices.Sort(names) + known := map[string]bool{} + for _, name := range names { + value, _, found, err := runs.Get(ctx, name) + if err != nil { + return out, fmt.Errorf("%s.%s: %w", RunsState, name, err) + } + var r Run + if !found || json.Unmarshal(value, &r) != nil { + continue + } + key, _ := meshcatalogclient.CandidatesKey(name) + known[key] = true + read := RunRead{Run: r.Run, Group: name, Members: r.Members, Ended: r.Ended, At: r.At, Candidate: held[key]} + for _, m := range r.Members { + if !slices.Contains(r.Ended, m.Module) { + read.Waiting = append(read.Waiting, m.Module) + } + } + out.Runs = append(out.Runs, read) + } + for k := range held { + if group, err := meshcatalogclient.ParseCandidatesKey(k); err == nil && strings.HasPrefix(group, "run-") && !known[k] { + out.Orphans = append(out.Orphans, group) + } + } + slices.Sort(out.Orphans) + return out, nil +} + +// runGroupOf is the group of a run the tool names: the runner's id, or run- as indexer_runs lists it. +func runGroupOf(run string) (string, error) { + return RunGroup(strings.TrimPrefix(strings.TrimSpace(run), "run-")) +} + +// DropRunByHand is indexer_run_drop answered: the run's candidate and its key in the state runs are removed, as they are when +// every member's pipeline has ended. A run no state or candidate holds drops nothing. +func (ix *Indexer) DropRunByHand(ctx context.Context, run, by, why string) TakenIn { + r := TakenIn{Module: "run " + run} + group, err := runGroupOf(run) + if err != nil { + r.Answer = err.Error() + ": nothing dropped" + return r + } + if ix.Runs == nil { + r.Answer = "the indexer keeps no state " + RunsState + ": nothing dropped" + return r + } + ix.runMu.Lock() + defer ix.runMu.Unlock() + read, err := ReadRuns(ctx, ix.Runs, ix.Store) + if err != nil { + r.Answer = "the runs are not read: " + err.Error() + return r + } + if !slices.ContainsFunc(read.Runs, func(x RunRead) bool { return x.Group == group }) && !slices.Contains(read.Orphans, group) { + r.Answer = "no run " + group + " is held by the state " + RunsState + " or the candidates: nothing dropped" + return r + } + if err := ix.dropRun(ctx, group); err != nil { + r.Answer = group + " could not be dropped: " + err.Error() + ix.logf("%s", r.Answer) + return r + } + r.Stored = true + r.Answer = fmt.Sprintf("the run %s and its candidate are dropped by %s (%s)", group, by, why) + ix.logf("%s", r.Answer) + return r +} + +// RunsTools are indexer_runs and indexer_run_drop (class act): the tools bundle reads the states and writes an ask in takeins, +// and the indexer, the state's owner, drops the run; record says the hand act in the hand-act log before it is asked. +func RunsTools(runs, candidates Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time, record func(what, why string) error) []stdio.Tool { + return []stdio.Tool{ + {Name: RunsTool, + Description: "Every run on request the indexer holds a candidate for, from its state runs (design 47 G §64, A-174): the run's id and " + + "group, its members (module, commit, branch), the modules whose pipeline of the run has ended and those still waiting, " + + "when it was last written, and whether its candidate is in the state candidates; and the run candidates no run of the state " + + "holds (orphans). indexer_run_drop drops one.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + return ReadRuns(context.Background(), runs, candidates) + }}, + {Name: RunDropTool, + Description: "Drop one run on request of the indexer: its candidate and its key in the state runs, as the end of its last member's " + + "pipeline does (design 47 G §64, A-194), for a run whose pipelines never end. A hand act, recorded through mesh-hand-acts " + + "before it is asked. indexer_runs names the runs.", + Input: map[string]any{"type": "object", "required": []string{"run", "why"}, "properties": map[string]any{ + "run": map[string]any{"type": "string", "description": "the run's id, or its group run-, as indexer_runs names it"}, + "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + run, why := str("run"), str("why") + group, err := runGroupOf(run) + if err != nil || why == "" { + return nil, errors.New("name the run as indexer_runs names it (run) and say why (why): a drop is a hand act, recorded through mesh-hand-acts") + } + data, err := json.Marshal(TakeInAsk{V: 1, RunDrop: group, Why: why, By: stdio.CallerOf(args), At: now().UTC()}) + if err != nil { + return nil, err + } + if err := record("dropped the run "+group+" and its candidate by hand", why); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err) + } + a, err := ask("run-drop."+keys.EscapeName(group), data) + if err != nil { + return nil, err + } + return a.Result, nil + }}, + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 6e1ed4b8..7e581059 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -73,6 +73,8 @@ type TakeInAsk struct { UnpinRevision string `json:"unpin-revision,omitempty"` UnpinRecord string `json:"unpin-record,omitempty"` NamedMember string `json:"named-member,omitempty"` + // RunDrop asks one run on request, by its group run-, to be dropped with its candidate (indexer_run_drop, runs_tools.go, A-194). + RunDrop string `json:"run-drop,omitempty"` // Registry asks one of the registrations' acts (registrations.go): a trusted change of one, the verb registrations, // indexer_proposal_drop or indexer_seed_again (design 47 G §3, §4, §8, §667). Registry *RegistryAct `json:"registry,omitempty"` @@ -94,7 +96,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.UnpinRevision == "" && a.Registry == nil { + if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && !a.Fingerprints && !a.CatchUp && !a.Recompute && a.CandidateGroup == "" && a.NamedFingerprint == "" && a.UnpinRevision == "" && a.RunDrop == "" && a.Registry == nil { return a, errors.New("a takeins value naming no module, repository, pending wait or merge") } return a, nil diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index d1997018..2eba8cea 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -202,8 +202,8 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // milestone 47/G W4.12 (design 47, the indexer's tools): the catalog graph revisions read and steered. index.CatalogRevisionTool, index.CatalogRevisionsTool, index.CandidateGraphsTool, index.RecomputeTool, index.DropCandidateGraphTool, index.EntryIndexTool, index.NamedDropTool, // design 47 G §675 - index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool) // design 47 G §667, §8, §579 - if !reflect.DeepEqual(m.Tools, want) || len(want) != 47 || slices.Contains(m.Tools, "modules") { + index.ProposalsTool, index.ProposalDropTool, index.SeedAgainTool, index.InputsTool, index.PinsTool, index.UnpinTool, index.ConsumerTool, index.RunsTool, index.RunDropTool) // design 47 G §667, §8, §579, §64 (A-194) + if !reflect.DeepEqual(m.Tools, want) || len(want) != 49 || slices.Contains(m.Tools, "modules") { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 47 G §2, §9: every version of every registration is kept with no age limit, 64 MiB, on the old streams section. diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 1e7d2c6f..6041744f 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -73,7 +73,9 @@ "indexer_inputs", "indexer_pins", "indexer_unpin", - "indexer_consumer" + "indexer_consumer", + "indexer_runs", + "indexer_run_drop" ], "consumes": [ "go-package-registry.published", From e586c0a05c32916f6b41161f4ff3a1405d23793a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 06:23:13 +0100 Subject: [PATCH 08/24] A-238: a run ask whose program reads only its input names no head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The indexer's validation runs its manifest validator over its input alone (design 47 G §712: heads none); the holder lays an empty MESH_RUN_HEADS. Builds: design 47 G §712, A-238 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- modules/mesh-delivery-sdk/buildstep/ask_test.go | 5 +++-- modules/mesh-delivery-sdk/buildstep/run.go | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-sdk/buildstep/ask_test.go b/modules/mesh-delivery-sdk/buildstep/ask_test.go index f63c248b..7676d0f4 100644 --- a/modules/mesh-delivery-sdk/buildstep/ask_test.go +++ b/modules/mesh-delivery-sdk/buildstep/ask_test.go @@ -59,7 +59,9 @@ func stepAsk() StepAsk { // The ask the three build seats share is one neutral form: it round-trips, and every ill-formed ask is refused, // naming what is wrong (the rules of the old delivery.StepAsk, now the shared runner's). func TestAskCheckAndRoundTrip(t *testing.T) { - for _, a := range []StepAsk{goodBuild(), goodCheck(), runAsk(), stepAsk()} { + headless := runAsk() // a run whose program reads only its input names no head (design 47 G §712, A-238) + headless.Run.Heads = nil + for _, a := range []StepAsk{goodBuild(), goodCheck(), runAsk(), headless, stepAsk()} { raw, err := EncodeStepAsk(a) if err != nil { t.Fatal(err) @@ -106,7 +108,6 @@ func TestAskCheckAndRoundTrip(t *testing.T) { {"run at a short commit", func(a *StepAsk) { a.Run.Program.Commit = "0123456" }, "no full commit"}, {"run outside the repository", func(a *StepAsk) { a.Run.Program.Dir = "../x" }, "no clean directory"}, {"run of no package", func(a *StepAsk) { a.Run.Program.Package = "" }, "no package"}, - {"run over no head", func(a *StepAsk) { a.Run.Heads = nil }, "no head"}, {"run over a head twice", func(a *StepAsk) { a.Run.Heads = append(a.Run.Heads, a.Run.Heads[0]) }, "twice"}, {"run over a branch", func(a *StepAsk) { a.Run.Heads[1].Commit = "main" }, "novox/mesh-sdk at no full commit"}, {"input on a state", func(a *StepAsk) { a.Run.Input.Verb = "mesh-delivery-checker.inputs" }, "no seat:."}, diff --git a/modules/mesh-delivery-sdk/buildstep/run.go b/modules/mesh-delivery-sdk/buildstep/run.go index e975cdf4..c98c6262 100644 --- a/modules/mesh-delivery-sdk/buildstep/run.go +++ b/modules/mesh-delivery-sdk/buildstep/run.go @@ -31,7 +31,9 @@ type RunJob struct { // Program is where the program is built: the repository and full commit fetched, the Go module's directory in it and // the program's import path. The ask's Packages are the mesh package versions it builds against (ADR 0573 §2, §4). Program RunProgram `json:"program"` - // Heads are the commits the program reads, each laid read-only at its repository's path; a repository at most once. + // Heads are the commits the program reads, each laid read-only at its repository's path; a repository at most once. None + // is a program that reads only its input (the indexer's manifest validator, design 47 G §712, A-238): the directory + // RunHeadsEnv names is then empty. Heads []RunHead `json:"heads"` // Input names the program's input, kept by the asker by its digest and served on a verb of its seat. Input RunInput `json:"input"` @@ -150,9 +152,6 @@ func (r RunJob) check(id string) error { if p.Package == "" || strings.ContainsAny(p.Package, " \t\n") { return fmt.Errorf("buildstep: the run ask %s names no package to build (%q)", id, p.Package) } - if len(r.Heads) == 0 { - return fmt.Errorf("buildstep: the run ask %s names no head", id) - } seen := map[string]bool{} for _, h := range r.Heads { if err := checkRunRepository(id, "a head of", h.Repository); err != nil { From c76d8ef381da08f8e860b1310fcd9419b52459e5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 06:24:13 +0100 Subject: [PATCH 09/24] IX5: the validation's request, run and verdict over the change and trunk cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The running indexer's validation before take-in asks its own manifest validator (cmd/mesh-judge under the indexer entry's Go module, at the revision's head) over two cases and refuses only what the revision causes; mesh-judge reads the file MESH_RUN_INPUT names, where a run writes its input. Builds: design 47 G §712, A-213, A-232 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-judge/main.go | 10 +- .../cmd/mesh-judge/main_test.go | 37 +++ .../internal/index/seatsof.go | 11 +- .../internal/index/validate.go | 234 ++++++++++++++++++ .../internal/index/validate_test.go | 74 ++++++ 5 files changed, 363 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-indexer/cmd/mesh-judge/main_test.go create mode 100644 modules/mesh-delivery-indexer/internal/index/validate.go create mode 100644 modules/mesh-delivery-indexer/internal/index/validate_test.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-judge/main.go b/modules/mesh-delivery-indexer/cmd/mesh-judge/main.go index 0fb0713a..e0fab53e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-judge/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-judge/main.go @@ -1,7 +1,9 @@ // Command mesh-judge is the indexer's judge as a program the checker runs (novox/hq ADR 0578 §2): built at the trunk, or // at a group's mesh-delivery head when that head changes the judge, and run by a Go build module's compose step in its -// sandbox exactly as the composer is (ADR 0482 §4; delivery.ComposeJob with this package). It reads a composer request -// (mesh-sdk composer.Request; `-request `, or its standard input) and writes one line, a composer verdict, whose +// sandbox exactly as the composer is (ADR 0482 §4; delivery.ComposeJob with this package), and by the running indexer's +// validation before take-in as a build seat's run (design 47 G §712). It reads a composer request (mesh-sdk +// composer.Request; `-request `, else the file MESH_RUN_INPUT names, where a run writes its input, else its standard +// input) and writes one line, a composer verdict, whose // cases say of each manifest whether the judge accepts it and, when not, every problem. It reads nothing else: no bus, // store, network or environment. package main @@ -13,6 +15,7 @@ import ( "io" "os" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "mesh-delivery-indexer/judge" @@ -32,6 +35,9 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { } var raw []byte var err error + if *file == "" { + *file = os.Getenv(buildstep.RunInputEnv) + } if *file != "" { raw, err = os.ReadFile(*file) } else { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-judge/main_test.go b/modules/mesh-delivery-indexer/cmd/mesh-judge/main_test.go new file mode 100644 index 00000000..2fe499f5 --- /dev/null +++ b/modules/mesh-delivery-indexer/cmd/mesh-judge/main_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" +) + +// A run ask writes the request to the file MESH_RUN_INPUT names and passes no argument (design 47 G §712): the validator +// reads it there and answers both cases. +func TestTheRunsInputFileIsTheRequest(t *testing.T) { + req := composer.Request{V: composer.Version, Input: json.RawMessage("{}"), Cases: []composer.Case{ + {Name: "change", Manifests: map[string]composer.Manifest{}}, {Name: "trunk", Manifests: map[string]composer.Manifest{}}}} + raw, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + file := filepath.Join(t.TempDir(), "input") + if err := os.WriteFile(file, raw, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(buildstep.RunInputEnv, file) + var out, errs bytes.Buffer + if code := run(nil, strings.NewReader(""), &out, &errs); code != 0 { + t.Fatalf("exit %d: %s", code, errs.String()) + } + var v composer.Verdict + if err := json.Unmarshal(out.Bytes(), &v); err != nil || len(v.Cases) != 2 { + t.Fatalf("the verdict %s: %v", out.String(), err) + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/seatsof.go b/modules/mesh-delivery-indexer/internal/index/seatsof.go index 56c64c40..e32fcd5a 100644 --- a/modules/mesh-delivery-indexer/internal/index/seatsof.go +++ b/modules/mesh-delivery-indexer/internal/index/seatsof.go @@ -23,6 +23,15 @@ import ( // (the take-in is tried again); a definition that does not parse is said and left out, as the indexer that keeps it // would say it unparsed. func (ix *Indexer) revisionSeats(ctx context.Context, ready []PendingEntry) ([]catalogue.Seat, error) { + defs, err := ix.revisionSeatDefinitions(ctx, ready) + if err != nil || defs == nil { + return nil, err + } + return catalogue.SeatsFromDefinitions(slices.Collect(maps.Values(defs))), nil +} + +// revisionSeatDefinitions are the definitions revisionSeats reads, by seat; nil where the indexer has no forge reader. +func (ix *Indexer) revisionSeatDefinitions(ctx context.Context, ready []PendingEntry) (map[string]seatdef.Definition, error) { if ix.ReadFile == nil || ix.Files == nil { return nil, nil } @@ -48,5 +57,5 @@ func (ix *Indexer) revisionSeats(ctx context.Context, ready []PendingEntry) ([]c } } } - return catalogue.SeatsFromDefinitions(slices.Collect(maps.Values(defs))), nil + return defs, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/validate.go b/modules/mesh-delivery-indexer/internal/index/validate.go new file mode 100644 index 00000000..a394aa1e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/validate.go @@ -0,0 +1,234 @@ +package index + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "mesh-delivery-indexer/judge" +) + +// The validation before take-in (design 47 G §712, A-213, A-232): for a pending revision of a watched branch the running +// indexer asks the seat node-build-go a `run` of its own manifest validator over two cases, change and trunk, and refuses only +// what the revision causes (ADR 0579 §2). The ask rides the check queue under the indexer's name, its kind `run` in the body. +const ( + // validationProgram is the manifest validator's program, by its directory under the indexer entry's Go module: the indexer's + // own code, so no repository or module name is fixed here (ADR 0666 §5). + validationProgram = "cmd/mesh-judge" + // validationAsker is the asker the ask is put under. A-232 names the indexer's seat name mesh-catalog; the ask code still + // names askers by their nicknames until the asker rename after the join (A-163), so this is `indexer` until then. + validationAsker = "indexer" + // validationCatalogSeat is the seat the indexer entry claims. + validationCatalogSeat = "mesh-catalog" + // The cases of the request, as the manifest validator names them. + validationChange = "change" + validationTrunk = judge.TrunkCase +) + +// validationInputVerb is where the build seat's holder asks the input: the seat mesh-catalog's verb input (§354). +var validationInputVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbInput + +// validationRequest is the manifest validator's request for the ready merges of branch over the branch's current revision +// (§712): case change, every module of the revision by its entry's manifest, each one the merges carry replaced by the merged +// message's bytes (the newest merge governs) and one added for a new directory; case trunk, every module as it stands; the +// seat definitions the merges' trees carry; the packages and the predefined list as the take-in's own validator has them; +// no machines. It answers the indexer entry of the change case too, the one that claims the seat mesh-catalog. +func (ix *Indexer) validationRequest(ctx context.Context, branch string, ready []PendingEntry) (composer.Request, meshcatalog.CatalogEntry, error) { + var indexer meshcatalog.CatalogEntry + if ix.Revisions == nil { + return composer.Request{}, indexer, errors.New("the catalog graph is not kept here: no revision to validate over") + } + rev, found, err := newestOf(ctx, ix.Revisions, branch) + if err != nil { + return composer.Request{}, indexer, err + } + var entries []meshcatalog.CatalogEntry + if found { + if _, entries, err = ReadRevision(ctx, ix.Revisions, RevisionObject(rev)); err != nil { + return composer.Request{}, indexer, err + } + } + trunkCase := composer.Case{Name: validationTrunk, Manifests: map[string]composer.Manifest{}} + byPlace := map[string]string{} // -> module + of := map[string]meshcatalog.CatalogEntry{} + for _, e := range entries { + raw := ix.manifestBytes(ctx, e.ManifestDigest) + if raw == "" { + b, err := json.Marshal(e.Manifest) + if err != nil { + return composer.Request{}, indexer, fmt.Errorf("the manifest of %s: %w", e.Module, err) + } + raw = string(b) + } + trunkCase.Manifests[e.Module] = composer.Manifest{Raw: json.RawMessage(raw)} + byPlace[placeKey(e.Repository, e.Path)] = e.Module + of[e.Module] = e + } + change := composer.Case{Name: validationChange, Manifests: maps.Clone(trunkCase.Manifests)} + for _, e := range ready { + if e.Covered { + continue + } + for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) { + raw := e.Forge.Manifests[dir] + module, known := byPlace[placeKey(e.Forge.Repository, dir)] + if !known { + m, err := judge.ForInputs([]byte(raw)) + if err != nil || m.Module == "" { + module = e.Forge.Repository + "/" + dir // a new directory whose manifest names no module: judged under its place + } else { + module = m.Module + } + byPlace[placeKey(e.Forge.Repository, dir)] = module + of[module] = meshcatalog.CatalogEntry{Module: module, Repository: e.Forge.Repository, Path: dir} + } + change.Manifests[module] = composer.Manifest{Raw: json.RawMessage(raw)} + } + } + for _, module := range slices.Sorted(maps.Keys(change.Manifests)) { + m, err := judge.ForInputs(change.Manifests[module].Raw) + if err != nil || !slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == validationCatalogSeat }) { + continue + } + indexer = of[module] + if indexer.Module == "" { + indexer.Module = module + } + break + } + seats, err := ix.revisionDefinitions(ctx, ready) + if err != nil { + return composer.Request{}, indexer, err + } + req := composer.Request{V: composer.Version, Input: json.RawMessage("{}"), Cases: []composer.Case{change, trunkCase}, Seats: seats, + Predefined: ix.predefinedList(ctx)} + if known := ix.knownPackages(ctx); known != nil { + req.Packages = slices.Sorted(maps.Keys(known)) + } + return req, indexer, nil +} + +func placeKey(repository, path string) string { + return strings.ToLower(trunk.RepositoryName(repository)) + " " + path +} + +// revisionDefinitions are the seat definitions the merges' trees carry, raw (§712 `seats`). +func (ix *Indexer) revisionDefinitions(ctx context.Context, ready []PendingEntry) ([]json.RawMessage, error) { + defs, err := ix.revisionSeatDefinitions(ctx, ready) + if err != nil { + return nil, err + } + var out []json.RawMessage + for _, seat := range slices.Sorted(maps.Keys(defs)) { + raw, err := json.Marshal(defs[seat]) + if err != nil { + return nil, fmt.Errorf("the definition of %s: %w", seat, err) + } + out = append(out, raw) + } + return out, nil +} + +// validationJob is the run of the manifest validator for the request (§712): the program cmd/mesh-judge under the Go module +// the indexer entry's Go artifact names, at the entry's path, built at head, the revision's head of the entry's repository; no +// heads and no arguments; the input the request's bytes, served by the seat mesh-catalog's verb input. +func validationJob(indexer meshcatalog.CatalogEntry, head, branch string, seq uint64, input []byte) (buildstep.RunJob, error) { + var pkg string + for _, a := range indexer.Artifacts { + if a.Language == buildstep.AskGo && a.Package != nil && a.Package.Name != "" { + pkg = a.Package.Name + break + } + } + switch { + case indexer.Repository == "": + return buildstep.RunJob{}, fmt.Errorf("no entry of the revision claims the seat %s", validationCatalogSeat) + case pkg == "": + return buildstep.RunJob{}, fmt.Errorf("the indexer entry %s names no Go artifact's package", indexer.Module) + case head == "": + return buildstep.RunJob{}, fmt.Errorf("the revision names no head of %s, the repository of the indexer entry %s", indexer.Repository, indexer.Module) + } + in, err := buildstep.RunInputOf(validationInputVerb, input) + if err != nil { + return buildstep.RunJob{}, err + } + dir := indexer.Path + if dir == "" { + dir = "." + } + return buildstep.RunJob{ + Program: buildstep.RunProgram{Repository: indexer.Repository, Commit: head, Dir: dir, Package: pkg + "/" + validationProgram}, + Input: in, + Reader: fmt.Sprintf("manifest validator of %s at %d", branchName(branch), seq), + }, nil +} + +func branchName(branch string) string { + if isTrunkBranch(branch) { + return TrunkBranch + } + return branch +} + +// ValidationRefusal is what a validation's verdict refuses (ADR 0579 §2, §712): of each manifest the change case refuses that +// the revision adds or changes (its bytes differ from the trunk case's), each problem the trunk case lacks; and each problem +// between manifests the trunk case lacks. A problem only in unchanged manifests refuses nothing. It is empty when the revision +// is accepted. An answer that holds not both cases is no verdict: it answers an error, so the run could not validate. +func ValidationRefusal(req composer.Request, v composer.Verdict) (map[string][]string, error) { + cases := map[string]composer.CaseVerdict{} + for _, c := range v.Cases { + cases[c.Name] = c + } + change, okChange := cases[validationChange] + trunkV, okTrunk := cases[validationTrunk] + if !okChange || !okTrunk { + return nil, fmt.Errorf("the manifest validator's answer holds not both cases %s and %s", validationChange, validationTrunk) + } + raws := map[string]map[string]string{} + for _, c := range req.Cases { + raws[c.Name] = map[string]string{} + for name, m := range c.Manifests { + raws[c.Name][name] = InputDigest(m.Raw) + } + } + refused := map[string][]string{} + for _, module := range slices.Sorted(maps.Keys(change.Manifests)) { + r := change.Manifests[module] + if r.Accepted || r.Refused == "" { + continue + } + before, had := raws[validationTrunk][module] + if module != judge.Unnamed && had && before == raws[validationChange][module] { + continue // unchanged: its problems refuse nothing, and its entry keeps its verdict (ADR 0579 §1) + } + old := map[string]bool{} + for _, p := range strings.Split(trunkV.Manifests[module].Refused, "\n") { + old[p] = true + } + for _, p := range strings.Split(r.Refused, "\n") { + if p != "" && !old[p] { + refused[module] = append(refused[module], p) + } + } + } + return refused, nil +} + +// RefusedWords is a refusal as the pending value's `refused` says it: each manifest and its problems, one manifest a line. +func RefusedWords(refused map[string][]string) string { + var lines []string + for _, module := range slices.Sorted(maps.Keys(refused)) { + lines = append(lines, module+": "+strings.Join(refused[module], "; ")) + } + return strings.Join(lines, "\n") +} diff --git a/modules/mesh-delivery-indexer/internal/index/validate_test.go b/modules/mesh-delivery-indexer/internal/index/validate_test.go new file mode 100644 index 00000000..d99a3358 --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/validate_test.go @@ -0,0 +1,74 @@ +package index + +import ( + "encoding/json" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" + + "mesh-delivery-indexer/judge" +) + +// The validation refuses only what the revision causes (design 47 G §712, ADR 0579 §2), and its run names the indexer entry's +// repository, path and the revision's head, and no heads. +func TestValidationRefusesWhatTheRevisionCauses(t *testing.T) { + manifests := func(m map[string]string) map[string]composer.Manifest { + out := map[string]composer.Manifest{} + for k, v := range m { + out[k] = composer.Manifest{Raw: json.RawMessage(v)} + } + return out + } + req := composer.Request{Cases: []composer.Case{ + {Name: "change", Manifests: manifests(map[string]string{"a": `{"module":"a","v":2}`, "b": `{"module":"b"}`})}, + {Name: "trunk", Manifests: manifests(map[string]string{"a": `{"module":"a"}`, "b": `{"module":"b"}`})}, + }} + verdict := func(change, trunk map[string]string) composer.Verdict { + read := func(m map[string]string) map[string]composer.ManifestRead { + out := map[string]composer.ManifestRead{"a": {Accepted: true}, "b": {Accepted: true}} + for k, v := range m { + out[k] = composer.ManifestRead{Refused: v} + } + return out + } + return composer.Verdict{Cases: []composer.CaseVerdict{{Name: "change", Manifests: read(change)}, {Name: "trunk", Manifests: read(trunk)}}} + } + for _, c := range []struct { + name string + v composer.Verdict + want, wantErr string + }{ + {"a changed manifest gains a problem", verdict(map[string]string{"a": "new\nold"}, map[string]string{"a": "old"}), "a: new", ""}, + {"an unchanged manifest's problem refuses nothing", verdict(map[string]string{"b": "bad"}, nil), "", ""}, + {"a problem between manifests the trunk lacks", verdict(map[string]string{judge.Unnamed: "x\ny"}, map[string]string{judge.Unnamed: "y"}), judge.Unnamed + ": x", ""}, + {"an answer with one case is no verdict", composer.Verdict{Cases: verdict(nil, nil).Cases[:1]}, "", "not both cases"}, + } { + got, err := ValidationRefusal(req, c.v) + if c.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), c.wantErr) { + t.Errorf("%s: %v, want %q", c.name, err, c.wantErr) + } + continue + } + if err != nil || RefusedWords(got) != c.want { + t.Errorf("%s: %q, %v, want %q", c.name, RefusedWords(got), err, c.want) + } + } + head := strings.Repeat("a", 40) + entry := meshcatalog.CatalogEntry{Module: "mesh-delivery-indexer", Repository: "novox/mesh-delivery", Path: "modules/mesh-delivery-indexer", + Artifacts: []meshcatalog.Artifact{{Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "mesh-delivery-indexer"}}}} + job, err := validationJob(entry, head, "main", 7, []byte(`{"v":1}`)) + if err != nil { + t.Fatal(err) + } + want := buildstep.RunProgram{Repository: "novox/mesh-delivery", Commit: head, Dir: "modules/mesh-delivery-indexer", Package: "mesh-delivery-indexer/cmd/mesh-judge"} + if job.Program != want || len(job.Heads) != 0 || len(job.Args) != 0 || job.Input.Verb != "seat:mesh-catalog.input" || job.Reader != "manifest validator of main at 7" { + t.Errorf("the job %+v", job) + } + if _, err := validationJob(meshcatalog.CatalogEntry{}, head, "main", 7, nil); err == nil || !strings.Contains(err.Error(), "claims the seat mesh-catalog") { + t.Errorf("no indexer entry: %v", err) + } +} From b7e251a46394a6cf759cd7df0acd0681c43d7d55 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 06:27:42 +0100 Subject: [PATCH 10/24] B5g3-1: a run's input is any size, RunInputBound goes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §712, §354, A-237 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../cmd/mesh-checking-program/main_test.go | 14 +++++++------- modules/mesh-delivery-checker/rules/seats.go | 4 ++-- .../mesh-delivery-sdk/buildstep/ask_test.go | 7 ++++--- modules/mesh-delivery-sdk/buildstep/run.go | 18 +++++------------- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/modules/mesh-delivery-checker/cmd/mesh-checking-program/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-checking-program/main_test.go index cbd54007..972fa7d8 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-checking-program/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-checking-program/main_test.go @@ -153,8 +153,8 @@ func TestTheCheckingProgramAnswersTheVerdictOfItsFacts(t *testing.T) { } // Review finding 3 of milestone 47/G: the heads of a real group carry every seat definition the reader holds, unchanged // (this repository's own contracts, laid as its head): the answer leaves them all to the checker's list and stays - // within the bound a run's answer is written under (buildstep.RunInputBound), where the whole list would not. - t.Run("heads carrying every definition the reader holds unchanged answer none of them, within the bound", func(t *testing.T) { + // far below the whole list. + t.Run("heads carrying every definition the reader holds unchanged answer none of them", func(t *testing.T) { real := under{prefix: "novox/mesh-delivery", fs: os.DirFS("../../../..")} group := rules.Facts{V: rules.FactsVersion, Members: []rules.Member{{Repository: "novox/mesh-delivery", Head: "h"}}} // An empty list of digests answers every definition whole (a checker that hands its list's digests, holding none). @@ -166,9 +166,9 @@ func TestTheCheckingProgramAnswersTheVerdictOfItsFacts(t *testing.T) { group.SeatDigests = wholeDigests(t, real, "novox/mesh-delivery") same, err := rules.SeatsOf(real, group) answer, _ := json.Marshal(same) - if err != nil || len(same.Seats) != 0 || same.Same != len(whole.Seats) || len(answer) > buildstep.RunInputBound { - t.Errorf("unchanged heads answer %d definitions (%d the same), %d bytes; want none, all %d the same, within %d: %v", - len(same.Seats), same.Same, len(answer), len(whole.Seats), buildstep.RunInputBound, err) + if err != nil || len(same.Seats) != 0 || same.Same != len(whole.Seats) { + t.Errorf("unchanged heads answer %d definitions (%d the same), %d bytes; want none, all %d the same: %v", + len(same.Seats), same.Same, len(answer), len(whole.Seats), err) } // Review round 2 of G: a checker from before G (B0's, which makes G's own check) hands no digests, and the answer // then keeps each seat's own description only, so a group's whole list stays within the bound; each still parses. @@ -206,8 +206,8 @@ func TestTheCheckingProgramAnswersTheVerdictOfItsFacts(t *testing.T) { t.Errorf("B0-shaped facts answer %s with %d inner descriptions, its own %q: %v; want only its own", raw[:min(len(raw), 80)], inner, d.Description, perr) } } - if answer, _ := json.Marshal(b0); len(answer) > buildstep.RunInputBound || len(answer) >= len(mustJSON(t, whole))*3/4 { - t.Errorf("B0-shaped facts answer %d bytes (whole %d), want within %d and far below the whole", len(answer), len(mustJSON(t, whole)), buildstep.RunInputBound) + if answer, _ := json.Marshal(b0); len(answer) >= len(mustJSON(t, whole))*3/4 { + t.Errorf("B0-shaped facts answer %d bytes (whole %d), want far below the whole", len(answer), len(mustJSON(t, whole))) } // One definition the heads change is answered alone. for seat := range group.SeatDigests { diff --git a/modules/mesh-delivery-checker/rules/seats.go b/modules/mesh-delivery-checker/rules/seats.go index 1e3a37c8..50040e14 100644 --- a/modules/mesh-delivery-checker/rules/seats.go +++ b/modules/mesh-delivery-checker/rules/seats.go @@ -21,8 +21,8 @@ import ( // milestones 47/B0 and 47/G): the checking program built from the heads reads the definitions the heads' contracts // modules carry, by the domain SDK's one reading (sdk/seatdefs), the indexer's too, and answers them; the running checker // lays them by seat over the list its reader holds from the seat mesh-catalog (verb seat-definitions) and reads no file -// of the heads. The reader's list is not in the program's input: the mesh's definitions are larger than a run input's -// bound (buildstep.RunInputBound). +// of the heads. The reader's list is not in the program's input: it is large, and the answer carries only what the heads +// change. // SeatsVersion is the version of SeatsAnswer this program writes. const SeatsVersion = 1 diff --git a/modules/mesh-delivery-sdk/buildstep/ask_test.go b/modules/mesh-delivery-sdk/buildstep/ask_test.go index 7676d0f4..258b5215 100644 --- a/modules/mesh-delivery-sdk/buildstep/ask_test.go +++ b/modules/mesh-delivery-sdk/buildstep/ask_test.go @@ -113,7 +113,7 @@ func TestAskCheckAndRoundTrip(t *testing.T) { {"input on a state", func(a *StepAsk) { a.Run.Input.Verb = "mesh-delivery-checker.inputs" }, "no seat:."}, {"input on one machine", func(a *StepAsk) { a.Run.Input.Verb = "seat:mesh-delivery-checker.input@ace" }, "no seat:."}, {"input by no digest", func(a *StepAsk) { a.Run.Input.Digest = "d1" }, "no sha256 digest"}, - {"input over its bound", func(a *StepAsk) { a.Run.Input.Size = RunInputBound + 1 }, "outside 1 to"}, + {"input of no bytes", func(a *StepAsk) { a.Run.Input.Size = 0 }, "under 1"}, {"argument of the shell", func(a *StepAsk) { a.Run.Args = []string{"-x; rm"} }, "no word or flag"}, } { a := runAsk() @@ -148,8 +148,9 @@ func TestAskCheckAndRoundTrip(t *testing.T) { if a := stepAsk(); !a.Step.Skips("./modules/nats/a", "sha256:bbbb") || a.Step.Skips("./modules/nats/a", "sha256:cccc") { t.Error("a kept result skips its package at its digest and level only") } - if _, err := RunInputOf("seat:mesh-delivery-checker.input", make([]byte, RunInputBound+1)); err == nil || !strings.Contains(err.Error(), "262145 bytes") { - t.Errorf("an input over its bound: %v", err) + // A-237: a run's input is any size. + if in, err := RunInputOf("seat:mesh-delivery-checker.input", make([]byte, 3<<20)); err != nil || in.Size != 3<<20 { + t.Errorf("an input of 3 MiB: %+v, %v", in, err) } in := runAsk().Run.Input if err := in.Verify([]byte(`{"v":1}`)); err != nil { diff --git a/modules/mesh-delivery-sdk/buildstep/run.go b/modules/mesh-delivery-sdk/buildstep/run.go index c98c6262..c69879b7 100644 --- a/modules/mesh-delivery-sdk/buildstep/run.go +++ b/modules/mesh-delivery-sdk/buildstep/run.go @@ -20,8 +20,8 @@ import ( // Input names, checks its digest and writes it to the file RunInputEnv names; then runs the program with Args, no // network and no bus account (ADR 0448 §2, §4), and answers RunAnswer on its step key. // -// The input is carried by reference, as a compose carries its manifests: the asker keeps it by its digest, measured -// against RunInputBound before it asks, and serves it on a verb of its seat (RunInput.Verb); the build module asks that +// The input is carried by reference, as a compose carries its manifests: the asker keeps it by its digest, of any +// size (A-237), and serves it on a verb of its seat (RunInput.Verb); the build module asks that // verb as its manifest's `invokes` grants it, never reading the asker's state (ADR 0596 §2), so the ask on the stream // never carries it (issue 1930). The layouts are the part build-step's `RunJob`, `RunProgram`, `RunHead`, `RunInput` // and `RunAnswer`. @@ -77,22 +77,14 @@ type RunAnswer struct { Said string `json:"said,omitempty"` } -// RunInputBound is the most bytes a run's input may hold: the bus's value limit the design already uses (256 KiB), so -// the asker keeps it as one value (the lead's ruling on milestone 47/B0, 2026-11-06). -const RunInputBound = 256 << 10 - // RunHeadsEnv names, for the program, the directory its heads are laid in; RunInputEnv the file its input is written to. const ( RunHeadsEnv = "MESH_RUN_HEADS" RunInputEnv = "MESH_RUN_INPUT" ) -// RunInputOf is the reference to body served on verb (`seat:.`), refusing a body over RunInputBound, naming -// its size, so an asker refuses it before asking. +// RunInputOf is the reference to body served on verb (`seat:.`); a body is of any size. func RunInputOf(verb string, body []byte) (RunInput, error) { - if len(body) > RunInputBound { - return RunInput{}, fmt.Errorf("buildstep: a run's input of %d bytes is over its bound of %d bytes", len(body), RunInputBound) - } sum := sha256.Sum256(body) in := RunInput{Verb: verb, Digest: hex.EncodeToString(sum[:]), Size: len(body)} return in, in.check("") @@ -118,8 +110,8 @@ func (i RunInput) check(id string) error { return fmt.Errorf("buildstep: the run ask %s serves its input on %q, which is no seat:. of no one machine", id, i.Verb) case !runDigestOK.MatchString(i.Digest): return fmt.Errorf("buildstep: the run ask %s names its input by %q, which is no sha256 digest", id, i.Digest) - case i.Size <= 0 || i.Size > RunInputBound: - return fmt.Errorf("buildstep: the run ask %s names an input of %d bytes, outside 1 to %d", id, i.Size, RunInputBound) + case i.Size <= 0: + return fmt.Errorf("buildstep: the run ask %s names an input of %d bytes, under 1", id, i.Size) } return nil } From ccbefc5653dfd039b798e1f09a053fa4cdc7f91b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 09:14:28 +0100 Subject: [PATCH 11/24] B5g3-2: the mesh-catalog seat's input verb answers in chunks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §712, §354, §681, A-237, A-139 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- .../go/seats/meshcatalog/contract.go | 6 ++- .../go/seats/meshcatalog/contract_test.go | 4 +- .../seats/mesh-catalog/fixtures/verbs.json | 7 ++-- .../seats/mesh-catalog/seat.json | 38 +++++++++++-------- .../ts/seats/meshcatalog/index.d.ts | 6 ++- 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go index 504752fc..74876e63 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract.go @@ -1379,16 +1379,18 @@ type OutdatedAnswer struct { // InputInput is what input takes. type InputInput struct { + // 128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements + Call string `json:"call"` // the input's sha256 Digest string `json:"digest"` } // InputAnswer is what input answers. type InputAnswer struct { + // how many chunks are sent; 0 when there is no input of that digest + Chunks int64 `json:"chunks"` // the input's sha256 Digest string `json:"digest"` - // the input's exact bytes, base64 - Input string `json:"input"` } // EntryInput is what entry takes. diff --git a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go index b3ad08e4..d9e87f7c 100644 --- a/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go +++ b/modules/mesh-delivery-contracts/go/seats/meshcatalog/contract_test.go @@ -171,9 +171,9 @@ func TestFixtures(t *testing.T) { decode: func(b []byte) (any, error) { var v OutdatedInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "outdated answers an entry behind", subject: "", given: "{\"behind\":[{\"act\":\"rebuild\",\"dependency\":{\"artifact\":\"mesh-sdk:go\",\"branch\":\"main\",\"source\":3589},\"kind\":1,\"module\":\"nats\",\"newest\":\"0.1.15 sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"recorded\":\"0.1.14 sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"version\":\"1.2.3\"}]}", wire: "{\"behind\":[{\"act\":\"rebuild\",\"dependency\":{\"artifact\":\"mesh-sdk:go\",\"branch\":\"main\",\"source\":3589},\"kind\":1,\"module\":\"nats\",\"newest\":\"0.1.15 sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"recorded\":\"0.1.14 sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"version\":\"1.2.3\"}]}", decode: func(b []byte) (any, error) { var v OutdatedAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, - {name: "input asks a step's input by digest", subject: "", given: "{\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", wire: "{\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", + {name: "input asks a step's input by digest", subject: "", given: "{\"call\":\"0123456789abcdef0123456789abcdef\",\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", wire: "{\"call\":\"0123456789abcdef0123456789abcdef\",\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", decode: func(b []byte) (any, error) { var v InputInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, - {name: "input answers the exact bytes", subject: "", given: "{\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"input\":\"eyJ2IjoxfQ==\"}", wire: "{\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\",\"input\":\"eyJ2IjoxfQ==\"}", + {name: "input answers how many chunks carry the exact bytes", subject: "", given: "{\"chunks\":3,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", wire: "{\"chunks\":3,\"digest\":\"0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", decode: func(b []byte) (any, error) { var v InputAnswer; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, {name: "entry asks by fingerprint", subject: "", given: "{\"fingerprint\":\"src2:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", wire: "{\"fingerprint\":\"src2:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\"}", decode: func(b []byte) (any, error) { var v EntryInput; err := json.Unmarshal(b, &v); return v, err }, subjectIs: nil}, diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/verbs.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/verbs.json index 9b84c76a..e5f0651a 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/verbs.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/fixtures/verbs.json @@ -424,15 +424,16 @@ "name": "input asks a step's input by digest", "of": "verb:input.input", "given": { + "call": "0123456789abcdef0123456789abcdef", "digest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" } }, { - "name": "input answers the exact bytes", + "name": "input answers how many chunks carry the exact bytes", "of": "verb:input.answer", "given": { - "digest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - "input": "eyJ2IjoxfQ==" + "chunks": 3, + "digest": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" } }, { diff --git a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json index 0233171e..8c9841db 100644 --- a/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json +++ b/modules/mesh-delivery-contracts/seats/mesh-catalog/seat.json @@ -171,7 +171,7 @@ }, { "name": "seat-definitions", - "description": "The seat list (ADR 0636 \u00a74): with no seat, every trunk seat with its digest and the revision of the list; with a seat, that trunk definition resolved with the parts it includes. Reads only.", + "description": "The seat list (ADR 0636 §4): with no seat, every trunk seat with its digest and the revision of the list; with a seat, that trunk definition resolved with the parts it includes. Reads only.", "input": { "properties": { "seat": { @@ -250,7 +250,7 @@ }, { "name": "revision", - "description": "The catalog half of the revision view (ADR 0656 \u00a76): one catalog graph revision with its entries whole, the trunk heads, the seat definitions by digest, and each manifest's verdict and fingerprint; the trunk's newest when no revision and no candidate is named, or a candidate's for its group and heads. Answered by the holder from what it holds, in its own runtime (ADR 0663), in pages of entries. Reads only. A watched branch with no revision yet answers {none-yet: true} alone (design 47 G section 530).", + "description": "The catalog half of the revision view (ADR 0656 §6): one catalog graph revision with its entries whole, the trunk heads, the seat definitions by digest, and each manifest's verdict and fingerprint; the trunk's newest when no revision and no candidate is named, or a candidate's for its group and heads. Answered by the holder from what it holds, in its own runtime (ADR 0663), in pages of entries. Reads only. A watched branch with no revision yet answers {none-yet: true} alone (design 47 G section 530).", "input": { "type": "object", "properties": { @@ -554,17 +554,23 @@ "seat:node-build-node", "seat:node-build-image" ], - "description": "A step's input by its digest, as the indexer keeps it for a validation step it asked a build seat to run (design 47 G section 354).", + "description": "A step's input by its digest, as the indexer keeps it for a validation step it asked a build seat to run (design 47 G section 354): its fields, then the bytes in chunks of at most 512 KiB on the chunk subject of the call, each acknowledged by the caller on the ack subject within 30 seconds; the input is of any size.", "input": { "type": "object", "properties": { "digest": { "type": "string", "description": "the input's sha256" + }, + "call": { + "type": "string", + "description": "128 random bits the caller chooses: the subject token of this send's chunks and acknowledgements", + "minLength": 1 } }, "required": [ - "digest" + "digest", + "call" ] }, "answer": { @@ -574,16 +580,18 @@ "type": "string", "description": "the input's sha256" }, - "input": { - "type": "string", - "description": "the input's exact bytes, base64" + "chunks": { + "type": "integer", + "minimum": 0, + "description": "how many chunks are sent; 0 when there is no input of that digest" } }, "required": [ "digest", - "input" + "chunks" ] - } + }, + "chunks": true }, { "name": "entry", @@ -614,7 +622,7 @@ } } ], - "description": "The catalog of the mesh: every module the trunk holds with its manifest, fingerprint and verdict, the packages published, and the seat list (ADR 0636 \u00a71, \u00a74).", + "description": "The catalog of the mesh: every module the trunk holds with its manifest, fingerprint and verdict, the packages published, and the seat list (ADR 0636 §1, §4).", "family": { "name": "mesh-catalog.events", "max-age-days": 7 @@ -1793,7 +1801,7 @@ "name": "inputs", "template": "state", "history": 1, - "description": "a step's input, named by its write time and its sha256 (design 47 G section 354)", + "description": "a step's input, named by its write time and its sha256; a value over one chunk is held in the state's objects store (design 47 G sections 354, 712)", "key-forms": [ { "key": "