From 496989ccfb485488bf9a6c42d61d0409ad1d2adc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:43:53 +0100 Subject: [PATCH 01/16] CT10: meshcatalogclient follows the registrations, a first view then each registration-changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assigner and the forge module's merge switch on their first registrations (design 47 G §353), so the seat's client carries the typed follower on go/views: a first view asked of the verb registrations, a newer version heard asked again until the picture holds it, an answer read before the change asked once more. Builds: design 47 G §353, §245 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- contracts/go/clients/meshcatalogclient/go.mod | 9 + contracts/go/clients/meshcatalogclient/go.sum | 26 ++- .../meshcatalogclient/registrations.go | 200 ++++++++++++++++++ .../meshcatalogclient/registrations_test.go | 140 ++++++++++++ 4 files changed, 369 insertions(+), 6 deletions(-) create mode 100644 contracts/go/clients/meshcatalogclient/registrations.go create mode 100644 contracts/go/clients/meshcatalogclient/registrations_test.go diff --git a/contracts/go/clients/meshcatalogclient/go.mod b/contracts/go/clients/meshcatalogclient/go.mod index 450b54a2..433b7918 100644 --- a/contracts/go/clients/meshcatalogclient/go.mod +++ b/contracts/go/clients/meshcatalogclient/go.mod @@ -6,3 +6,12 @@ require ( git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 ) + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/contracts/go/clients/meshcatalogclient/go.sum b/contracts/go/clients/meshcatalogclient/go.sum index 0daef11f..48737795 100644 --- a/contracts/go/clients/meshcatalogclient/go.sum +++ b/contracts/go/clients/meshcatalogclient/go.sum @@ -1,12 +1,26 @@ -github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= -github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= -github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= diff --git a/contracts/go/clients/meshcatalogclient/registrations.go b/contracts/go/clients/meshcatalogclient/registrations.go new file mode 100644 index 00000000..d58aaf0e --- /dev/null +++ b/contracts/go/clients/meshcatalogclient/registrations.go @@ -0,0 +1,200 @@ +package meshcatalogclient + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// RegistrationsVerb is the seat verb a part takes its first view of the registrations by, as its manifest's invokes +// names it. +const RegistrationsVerb = "seat:" + seat.Seat + "." + seat.VerbRegistrations + +// Call asks one seat verb over the bus and answers what the holder said, whatever wraps it (process.Conn.Call, or +// stdio.Ask through the tool runner). +type Call func(key string, body any) (json.RawMessage, error) + +// registrationsCreated stands for the picture's Created in its follower: the registrations are followed by the seat's +// event, never by an owner's views family, so one fixed time, never zero. +var registrationsCreated = time.Unix(0, 0).UTC() + +// Registrations is a part's typed follower of the registrations (design 47 G §353): a first view asked of the seat's verb +// `registrations`, then each `registration-changed` heard, built on go/views' Follower, so the runtime's tools +// `_first_views` and `_first_view_again` read and steer it as every first view. Its picture is keyed by +// module, each value the module's newest registration. Ready is closed at the first view: a part that switches on its +// first registrations waits for it. +type Registrations struct { + Follower *views.Follower + + call Call +} + +// NewRegistrations is a Registrations asking the seat through call. Nothing is asked before Start. +func NewRegistrations(call Call) *Registrations { + r := &Registrations{call: call} + r.Follower = &views.Follower{Owner: seat.Seat, Tool: RegistrationsVerb, State: seat.StateRegistrations, Ask: r.ask} + return r +} + +// WireRegistrations is a Registrations adopted by a stdio part's views.Reader: the reader writes its conditions, reads it +// as `mesh-catalog.registrations`, lists and asks it again in its first-view tools, and at its Start subscribes the +// seat's event registration-changed (the part's manifest consumes it) into Hear and asks the first view. changed, when +// not nil, is told of each module whose registration the picture changes, after the reader's own. The part declares the +// seat's verb registrations in invokes. +func WireRegistrations(r *views.Reader, call Call, changed func(reg seat.Registration, gone bool)) (*Registrations, error) { + g := NewRegistrations(call) + g.Follower.Changed = typedChanged(changed) + if err := r.Adopt(g.Follower, seat.Seat, seat.EventRegistrationChanged, g.Hear); err != nil { + return nil, err + } + return g, nil +} + +// WireRegistrationsProcess is WireRegistrations for a process's views.ProcessReader: its picture +// (`mesh-catalog_registrations`) reads the registrations as a read-only bucket, and the seat's event +// registration-changed on the module's durable consumer goes to Hear. +func WireRegistrationsProcess(r *views.ProcessReader, call Call, changed func(reg seat.Registration, gone bool)) (*Registrations, *views.Picture, error) { + g := NewRegistrations(call) + g.Follower.Changed = typedChanged(changed) + p, err := r.Adopt(g.Follower, seat.Seat, seat.EventRegistrationChanged, g.Hear) + if err != nil { + return nil, nil, err + } + return g, p, nil +} + +func typedChanged(changed func(seat.Registration, bool)) func(string, json.RawMessage, bool, time.Time) { + if changed == nil { + return nil + } + return func(module string, value json.RawMessage, gone bool, _ time.Time) { + reg := seat.Registration{Module: module} + if !gone { + _ = json.Unmarshal(value, ®) + } + changed(reg, gone) + } +} + +// Start asks the first view until it is answered or ctx ends. +func (r *Registrations) Start(ctx context.Context) error { return r.Follower.Start(ctx) } + +// Ready is closed once the first view is answered. +func (r *Registrations) Ready() <-chan struct{} { return r.Follower.Ready() } + +// Registration is one module's newest registration in the picture. +func (r *Registrations) Registration(module string) (seat.Registration, bool) { + raw, ok := r.Follower.Get(module) + if !ok { + return seat.Registration{}, false + } + var reg seat.Registration + if err := json.Unmarshal(raw, ®); err != nil { + return seat.Registration{}, false + } + return reg, true +} + +// All is every registration in the picture, by module name. +func (r *Registrations) All() []seat.Registration { + values := r.Follower.Values() + out := make([]seat.Registration, 0, len(values)) + for _, raw := range values { + var reg seat.Registration + if json.Unmarshal(raw, ®) == nil { + out = append(out, reg) + } + } + slices.SortFunc(out, func(a, b seat.Registration) int { + if a.Module < b.Module { + return -1 + } + if a.Module > b.Module { + return 1 + } + return 0 + }) + return out +} + +// Hear takes one `registration-changed` event, as the part's durable consumer of the seat's events hands it. A version +// the picture holds, or one older, is passed over; a newer one asks the first view again on its own goroutine, so the +// consumer is never held while the holder is away, and again until the picture holds that version or ctx ends (an ask +// already running, the first view among them, may have been answered before the change was written). Data that is no such event is an error. +func (r *Registrations) Hear(ctx context.Context, data []byte) error { + var ev seat.RegistrationChangedEvent + if err := json.Unmarshal(data, &ev); err != nil || ev.Module == "" || ev.Version <= 0 { + return fmt.Errorf("meshcatalogclient: not a %s event: %s", seat.EventRegistrationChanged, clipBytes(data)) + } + if r.holds(ev.Module, ev.Version) { + return nil + } + go r.catchUp(ctx, ev.Module, ev.Version) + return nil +} + +func (r *Registrations) holds(module string, version int64) bool { + reg, ok := r.Registration(module) + return ok && reg.Version >= version +} + +// catchUp asks the first view again until the picture holds version of module, pausing between asks as the follower's +// Wait says (doubling to a minute), or until ctx ends. +func (r *Registrations) catchUp(ctx context.Context, module string, version int64) { + wait := r.Follower.Wait + if wait == 0 { + wait = 5 * time.Second + } + for !r.holds(module, version) { + if err := r.Follower.Gap(ctx, fmt.Sprintf("registration of %s at version %d said", module, version)); err != nil { + return + } + if r.holds(module, version) { + return + } + p, cancel := context.WithTimeout(ctx, wait) + <-p.Done() + cancel() + if ctx.Err() != nil { + return + } + if wait *= 2; wait > time.Minute { + wait = time.Minute + } + } +} + +// ask is the Follower's Ask: every registration, whole in one answer. +func (r *Registrations) ask(_ context.Context, _ string) (views.Answer, error) { + raw, err := r.call(RegistrationsVerb, seat.RegistrationsInput{}) + if err != nil { + return views.Answer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return views.Answer{}, fmt.Errorf("%s answered: %s", RegistrationsVerb, said) + } + var a seat.RegistrationsAnswer + if err := json.Unmarshal(answer, &a); err != nil || a.Registrations == nil { + return views.Answer{}, fmt.Errorf("%s answered no registrations: %s", RegistrationsVerb, clipBytes(answer)) + } + values := make(map[string]json.RawMessage, len(a.Registrations)) + for _, reg := range a.Registrations { + v, _ := json.Marshal(reg) + values[reg.Module] = v + } + return views.Answer{V: 1, Owner: seat.Seat, State: seat.StateRegistrations, Created: registrationsCreated, Values: values, At: time.Now()}, nil +} + +func clipBytes(b []byte) string { + if len(b) > 200 { + return string(b[:200]) + "…" + } + return string(b) +} diff --git a/contracts/go/clients/meshcatalogclient/registrations_test.go b/contracts/go/clients/meshcatalogclient/registrations_test.go new file mode 100644 index 00000000..66ff2546 --- /dev/null +++ b/contracts/go/clients/meshcatalogclient/registrations_test.go @@ -0,0 +1,140 @@ +package meshcatalogclient + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// fakeRegistrations answers the verb registrations from the versions it keeps, and counts each ask. +type fakeRegistrations struct { + mu sync.Mutex + versions map[string]int64 + asks int + // stale answers the next ask from these versions, once: an answer read before a change was written. + stale map[string]int64 +} + +func (f *fakeRegistrations) call(key string, body any) (json.RawMessage, error) { + if key != RegistrationsVerb { + return nil, fmt.Errorf("asked %s", key) + } + if in := body.(seat.RegistrationsInput); in.Module != "" || in.History { + return nil, fmt.Errorf("asked %+v", in) + } + f.mu.Lock() + defer f.mu.Unlock() + f.asks++ + versions := f.versions + if f.stale != nil { + versions, f.stale = f.stale, nil + } + a := seat.RegistrationsAnswer{Registrations: []seat.Registration{}} + for m, v := range versions { + a.Registrations = append(a.Registrations, seat.Registration{Module: m, Version: v, Origin: "novox/" + m, Path: "module.json", Branches: []string{"main"}}) + } + return json.Marshal(a) +} + +func (f *fakeRegistrations) set(module string, version int64) { + f.mu.Lock() + defer f.mu.Unlock() + f.versions[module] = version +} + +func (f *fakeRegistrations) asked() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.asks +} + +func changedEvent(module string, version int64) []byte { + b, _ := json.Marshal(seat.RegistrationChangedEvent{V: 1, Module: module, Version: version, Moved: []string{"branches"}, Warrant: "ask-1"}) + return b +} + +// CT10 (design 47 G §353): a part following the registrations takes a first view of the verb registrations, then each +// registration-changed: a newer version is asked and held, one it holds asks nothing, a change heard while an answer +// read before it was given asks again, and what is no such event is refused. +func TestRegistrationsAreAFirstViewThenTheirEvents(t *testing.T) { + changed := make(chan seat.Registration, 16) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + next := func() seat.Registration { + t.Helper() + select { + case reg := <-changed: + return reg + case <-ctx.Done(): + t.Fatal("no registration changed") + return seat.Registration{} + } + } + f := &fakeRegistrations{versions: map[string]int64{"nats": 1, "zsh": 3}} + r := NewRegistrations(f.call) + r.Follower.Wait = time.Nanosecond + r.Follower.Changed = typedChanged(func(reg seat.Registration, gone bool) { + if !gone { + changed <- reg + } + }) + select { + case <-r.Ready(): + t.Fatal("ready before its first view") + default: + } + if err := r.Start(ctx); err != nil { + t.Fatal(err) + } + <-r.Ready() + if reg, ok := r.Registration("nats"); !ok || reg.Version != 1 || reg.Origin != "novox/nats" { + t.Fatalf("first view holds nats as %+v, %v", reg, ok) + } + if all := r.All(); len(all) != 2 || all[0].Module != "nats" || all[1].Module != "zsh" { + t.Fatalf("All = %+v", all) + } + for range 2 { + next() + } + + if err := r.Hear(ctx, changedEvent("zsh", 3)); err != nil { + t.Fatal(err) + } + if n := f.asked(); n != 1 { + t.Errorf("a version held asked the first view again: %d asks", n) + } + + f.set("nats", 2) + if err := r.Hear(ctx, changedEvent("nats", 2)); err != nil { + t.Fatal(err) + } + if reg := next(); reg.Module != "nats" || reg.Version != 2 { + t.Fatalf("after registration-changed: %+v", reg) + } + + // The answer to the ask the event starts was read before the change was written: it is asked once more. + f.mu.Lock() + f.stale = map[string]int64{"nats": 2, "zsh": 3} + f.mu.Unlock() + f.set("zsh", 4) + if err := r.Hear(ctx, changedEvent("zsh", 4)); err != nil { + t.Fatal(err) + } + if reg := next(); reg.Module != "zsh" || reg.Version != 4 { + t.Fatalf("after a stale answer: %+v", reg) + } + if n := f.asked(); n != 4 { + t.Errorf("asks = %d, want 4 (first view, nats, a stale answer and the one after)", n) + } + + for _, data := range []string{`{}`, `not json`, `{"module":"nats"}`} { + if err := r.Hear(ctx, []byte(data)); err == nil { + t.Errorf("Hear(%s) took it", data) + } + } +} From 839bc4a6e5cbfe3866badcc090f559d3a7cec645 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:44:42 +0100 Subject: [PATCH 02/16] TL0: the teller's test follows the contracts, the settle ask gone with the verb settle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §431 (the settle ask goes with the verb settle) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-teller/teller.go | 4 ++-- .../cmd/mesh-delivery-teller/teller_test.go | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go index 83506c23..4426cb51 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go @@ -324,8 +324,8 @@ func (t *Teller) catchUp(id string) { t.said(h, *w, err) } -// mergeAct is what the approve option of the gate's ask binds: the ask's verb (meshdeliveryclient.AskVerb: mesh-delivery.merge, or -// mesh-delivery.settle for a person's settle ask, novox/hq ADR 0556 clause 5), this ask, the answer, and the heads it covers. +// mergeAct is what the approve option of the gate's ask binds: the ask's verb (meshdeliveryclient.AskVerb: mesh-delivery.merge; the settle ask +// went with the verb settle, design 47 G §431), this ask, the answer, and the heads it covers. // It is spelled as the gate composes it (mesh-catalog modules/mesh-delivery-gate approval.go mergeAct, from // modules/mesh-delivery approval.go:63-65) and as mesh-controller verifies it (cmd/mesh-controller/moved.go:98-106). func mergeAct(verb, ask, option string, heads map[string]string) occ.Act { diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go index 34758645..dac5305b 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller_test.go @@ -412,16 +412,15 @@ func TestAWarrantThatHoldsWritesOneAnswerAndASecondWritesNone(t *testing.T) { // if len(b.conds) != 0 { t.Errorf("conditions said: %+v", b.conds) } - // 1280 (ADR 0556 clause 5): a settle ask holds when its options bind mesh-delivery.settle, and not when they bind a merge. - for verb, holds := range map[string]bool{"mesh-delivery.settle": true, "mesh-delivery.merge": false} { + // design 47 G §431: the settle ask went with the verb settle, so an ask whose options bind it holds no more. + for verb, holds := range map[string]bool{"mesh-delivery.settle": false, "mesh-delivery.merge": true} { b := newBed(t) - group := meshdeliveryclient.SettleGroup([]string{"novox/mesh-delivery@" + headA[:12]}) - key, value, ga := gateAskOf(t, group, verb, twoHeads(), twoHeads(), 1) + key, value, ga := gateAskOf(t, "g", verb, twoHeads(), twoHeads(), 1) b.gate[key] = value b.t.OnAsk(key, 1, value) b.deliver(t, warrantFor(ga.Ask, "approve")) if _, written := b.answers.written[answerKey(key)]; written != holds { - t.Errorf("a settle ask binding %s: answer written %v, want %v", verb, written, holds) + t.Errorf("an ask binding %s: answer written %v, want %v", verb, written, holds) } } } From 74da8116db492e273e9f1b20a5c130ee03e88345 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:44:47 +0100 Subject: [PATCH 03/16] BL2: the build reserves its version at its start; reserved state, tools, verb and findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder keeps one key per module and MAJOR.MINOR in its state reserved, written by compare-and-set when a trunk build is claimed; nothing is reserved, and the build says wait-held, before the catalog says versions-filled; the key goes on version-released; a built reservation unpublished past ten minutes is the finding version-unreleased; builder_reserved, builder_reserved_drop (hand act, recorded) and the seat mesh-builds' verb reserved. Builds: design 47 G §369, §372, §52, §85 (reserved) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-builder-tools/main.go | 9 + .../mesh-delivery-builder-tools/reserved.go | 62 +++ .../mesh-delivery-builder/integration_test.go | 2 +- .../cmd/mesh-delivery-builder/main.go | 16 +- .../cmd/mesh-delivery-builder/reasks.go | 8 +- .../cmd/mesh-delivery-builder/reserve.go | 208 ++++++++ .../cmd/mesh-delivery-builder/reserve_test.go | 131 +++++ .../cmd/mesh-delivery-builder/seats.go | 5 + .../cmd/mesh-delivery-builder/versions.go | 9 +- .../internal/build/numbers.go | 57 +++ .../internal/build/plan.go | 8 +- .../internal/build/reserve_test.go | 104 ++++ .../internal/build/run.go | 42 +- .../internal/build/stuck.go | 21 + .../internal/versions/again.go | 2 + .../internal/versions/reserve.go | 450 ++++++++++++++++++ .../internal/versions/reserve_test.go | 158 ++++++ .../mesh-delivery-builder/manifest_test.go | 25 +- modules/mesh-delivery-builder/module.json | 16 + 19 files changed, 1306 insertions(+), 27 deletions(-) create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go create mode 100644 modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go create mode 100644 modules/mesh-delivery-builder/internal/build/reserve_test.go create mode 100644 modules/mesh-delivery-builder/internal/versions/reserve.go create mode 100644 modules/mesh-delivery-builder/internal/versions/reserve_test.go diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 002ecd3d..1411f36c 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -409,7 +409,16 @@ func main() { } return e.Value, true, nil }} + rs := stdio.State(versions.ReservedState) + rr := versions.Reader{Keys: rs.Keys, Get: func(key string) (json.RawMessage, bool, error) { + e, err := rs.Get(key) + if err != nil || e == nil { + return nil, false, err + } + return e.Value, true, nil + }} tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(builds.BuilderLog).Since, time.Now, vr, tags, tr, fr, ofr) + tools = append(tools, reservedTools(rr, buildAgain, time.Now, meshhandactsclient.Recorder{Ask: stdio.Ask})...) // The builder's expiring keys (state bounds): what each bounds and when it ends. tools = append(tools, stdio.Tool{Name: versions.BoundsTool, Description: versions.BoundsDescription, Input: map[string]any{"type": "object", "properties": map[string]any{}}, diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go new file mode 100644 index 00000000..8e3d342c --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/reserved.go @@ -0,0 +1,62 @@ +package main + +import ( + "errors" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + stdio "git.novox.be/novox/mesh-sdk/go" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// reservedTools are builder_reserved, builder_reserved_drop and the seat mesh-builds' verb reserved (design 47 G section +// 369), served as tools of the bundle. The drop is a hand act: it is recorded in the hand-act log first, then written to +// reasks for the builder, which owns the reservations, to answer. +func reservedTools(rr versions.Reader, ask queueAsker, now func() time.Time, acts meshhandactsclient.Recorder) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + return []stdio.Tool{ + {Name: versions.ReservedTool, Description: versions.ReservedDescription, + Input: map[string]any{"type": "object", "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "only this module's reservations"}}}, + Run: func(args map[string]any) (any, error) { return versions.AnswerReserved(rr, str(args, "module")) }}, + {Name: versions.ReservedVerb, Description: "The seat mesh-builds' verb reserved: a module's reserved version with each artifact's digest once built " + + "(design 47 G section 369), for a registry holder deciding whether to publish. Read-only.", + Input: map[string]any{"type": "object", "required": []string{"module"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module"}, + "version": map[string]any{"type": "string", "description": "the version, when asked for a particular one"}}}, + Run: func(args map[string]any) (any, error) { + return versions.AnswerReservedVerb(rr, str(args, "module"), str(args, "version")) + }}, + {Name: versions.ReservedDropTool, Description: versions.ReservedDropDescription, + Input: map[string]any{"type": "object", "required": []string{"module", "series", "why"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module"}, + "series": map[string]any{"type": "string", "description": "its MAJOR.MINOR"}, + "why": map[string]any{"type": "string", "description": "why, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + key, value, err := build.NewReservedDrop(str(args, "module"), str(args, "series"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return nil, err + } + if _, err := acts.Record(meshhandactsclient.Request{What: fmt.Sprintf("dropped the reservation of %s %s by hand", str(args, "module"), str(args, "series")), + Why: str(args, "why"), Cause: versions.ReservedDropTool, Verb: "reserved-drop"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was dropped: %w", err) + } + raw, err := ask(key, value) + if err != nil { + return nil, err + } + a, err := build.DecodeReAsk(raw) + if err != nil { + return nil, err + } + if a.Refused { + return nil, errors.New(a.Answer) + } + return map[string]any{"key": key, "answer": a.Answer}, nil + }}, + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 9c949810..6d270415 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -623,7 +623,7 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 } // Running-testing note (review round 1): two claims never get one number on the bus's own compare-and-set, as // the builder's state versions keeps them: eight claims of one name at once, eight versions. - store := versions.Versions{Store: awaitedVersions{process.AwaitBucket(b.ctx, b.js, nil, versions.Bucket, "", t.Logf)}} + store := versions.Versions{Store: awaitedVersions{b: process.AwaitBucket(b.ctx, b.js, nil, versions.Bucket, "", t.Logf)}} numbers := make(chan string, 8) var claims sync.WaitGroup for i := range 8 { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 5d40a676..659e7caa 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -163,7 +163,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro floorWaits.Keep(kvKeeper{ctx: ctx, kv: kv}, heldFloorWaits(ctx, kv, logger.Printf)) }) // A package is numbered under its name in its ecosystem, read from the indexer's packages state (design 32). - numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits, + numberer := &versions.Numberer{Versions: versions.Versions{Store: awaitedVersions{b: versionsKV}}, Floor: floorVia(calls.Call), Waits: floorWaits, Names: versions.NamesIn(indexerPackages{packages}), Said: func(nb builds.VersionNumbered) { if err := emitNumbered(context.WithoutCancel(ctx), js, cred, nb); err != nil { @@ -202,10 +202,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro boundsKV := process.AwaitBucket(ctx, js, conn, versions.TagAskBoundsBucket, "an unanswered tag ask is not narrated", logger.Printf) bounds := &tagBounds{asks: tagAsksKV.KV, bounds: boundsKV.KV, bound: tagAskBound, waits: floorWaits.Waiting, logf: logger.Printf, now: time.Now, log: func(e builds.BuildLogEntry) { b.Log(e) }} + // The versions a build reserves at its start (design 47 G §369): its state reserved, and the catalog's versions-filled + // and version-released heard; nothing is reserved before the fill. + reservedKV := process.AwaitBucket(ctx, js, conn, versions.ReservedBucket, "no version is reserved: builds are made unreserved", logger.Printf) + reserved := newReservations(awaitedReserved{awaitedVersions{b: reservedKV, bucket: versions.ReservedBucket}}, calls.Call, pubBoundsKV.KV, found, b.Reconsider, logger.Printf) + b.Reserved = reserved // A state bound later numbers the builds looked at without it. versionsKV.Ready(func(jetstream.KeyValue) { b.Reconsider(ctx) }) // The versions' hand acts (builder_publish_again, builder_floor_again) come through reasks, as builder_build_again. - acts := versions.Acts{Numberer: numberer, Done: doneVia(b), Reconsider: b.Reconsider, Remove: removeVia(calls.Call), + acts := versions.Acts{Numberer: numberer, DropReserved: reserved.dropped, Done: doneVia(b), Reconsider: b.Reconsider, Remove: removeVia(calls.Call), BuildAgain: func(ctx context.Context, module, fp, why, by string) (string, bool) { r := b.BuildAgain(ctx, module, fp, why, by, buildAgainBound) if r.Refused != "" { @@ -349,9 +354,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // condition keeper's condition-cleared. floor := floorWake{b: b, waits: floorWaits, logf: logger.Printf} // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. - seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs} + seats := seatHearer{n: numberer, b: b, logf: logger.Printf, pub: pubs, res: reserved} events := func(m jetstream.Msg) { - if reads.heard(m, logger.Printf) || floor.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) + if reads.heard(m, logger.Printf) || floor.heard(ctx, m) || reserved.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) return } heardGone(ctx, b, m, logger.Printf) @@ -407,6 +412,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) pubBoundsKV.Ready(func(kv jetstream.KeyValue) { each := func(e jetstream.KeyValueEntry) { + if reserved.ended(ctx, e) { + return + } if _, ok := versions.UnputBoundOf(e.Key()); ok { unput.expired(ctx, e) return diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go index 8d7c8d3c..bbcba722 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reasks.go @@ -53,7 +53,7 @@ func reAsked(ctx context.Context, b askAgainer, acts versions.Acts, kv jetstream announcedAgain(ctx, b, kv, e, r, logf) return } - if r.Mode == build.ModePublishAgain || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop { + if r.Mode == build.ModePublishAgain || r.Mode == build.ModeFloorAgain || r.Mode == build.ModeOldFormDrop || r.Mode == build.ModeReservedDrop { // A version's hand act (design 51 *Tools of the builder*) may build again, so it is answered beside the watch. go versionActed(ctx, acts, kv, e, r, logf) return @@ -163,6 +163,12 @@ func versionActed(ctx context.Context, acts versions.Acts, kv jetstream.KeyValue var ag versions.Again ag, r.Answer, err = acts.PublishAgain(ctx, r.Name, r.FP, r.Why, r.By, r.At) r.Do, r.Version = ag.Do, ag.Entry.Version + } else if r.Mode == build.ModeReservedDrop { + if acts.DropReserved == nil { + err = errors.New("this builder keeps no reservations") + } else { + r.Answer, err = acts.DropReserved(ctx, r.Name, r.FP, r.By, r.Why, r.At) + } } else if r.Mode == build.ModeOldFormDrop { if acts.DropOldForm == nil { err = errors.New("this builder keeps no old-form window") diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go new file mode 100644 index 00000000..bb41bbd8 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve.go @@ -0,0 +1,208 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "mesh-delivery-builder/internal/build" + "mesh-delivery-builder/internal/versions" +) + +// The reservations (design 47 G §369): the builder's state reserved, the catalog's versions it numbers from, and the +// catalog's version-released and versions-filled it hears. + +// awaitedReserved is the state reserved as versions.ReserveStore: bound once the bus holds it. +type awaitedReserved struct{ awaitedVersions } + +func (s awaitedReserved) Delete(ctx context.Context, key string) error { + kv, err := s.kv() + if err != nil { + return err + } + if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +// reservations is the builder's reserving: the Reserver, the fill the catalog said, the bounds of a built reservation's +// publish and the findings. +type reservations struct { + r *versions.Reserver + filled atomic.Bool + bounds func() jetstream.KeyValue + found *findings + bound time.Duration + wake func(context.Context) + logf func(string, ...any) +} + +// newReservations makes the reserving over store; the released versions of a module are asked of the seat mesh-catalog's +// verb versions through call (design 47 G §348), the only source numbering reads them from. +func newReservations(store versions.ReserveStore, call toolCaller, bounds func() jetstream.KeyValue, found *findings, wake func(context.Context), logf func(string, ...any)) *reservations { + rv := &reservations{bounds: bounds, found: found, bound: versions.UnreleasedBound, wake: wake, logf: logf} + rv.r = &versions.Reserver{Store: store, Filled: rv.filled.Load, ReleasedOf: func(_ context.Context, module string) ([]string, error) { + raw, err := call("seat:"+meshcatalog.Seat+".versions", meshcatalog.VersionsInput{Module: module}) + if err != nil { + return nil, err + } + var a meshcatalog.VersionsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the catalog's versions of %s answered what is no list of versions: %w", module, err) + } + var out []string + for _, v := range a.Versions { + out = append(out, v.Version) + } + return out, nil + }} + return rv +} + +var _ build.Reservations = (*reservations)(nil) + +func (rv *reservations) Filled() bool { return rv.filled.Load() } + +func (rv *reservations) Reserve(ctx context.Context, module, series string, artifacts []string) (versions.ReservedVersion, error) { + return rv.r.Reserve(ctx, module, series, artifacts) +} + +// Built records the digests; once every artifact has one, the publish's bound starts (a key of the state bounds with its +// own time to live): its end with the reservation standing is version-unreleased. +func (rv *reservations) Built(ctx context.Context, module, version string, digests map[string]string) error { + for ref, d := range digests { + if _, err := rv.r.Built(ctx, module, version, ref, d); err != nil { + return err + } + } + cur, found, err := rv.r.Of(ctx, module, version) + if err != nil || !found || cur.BuiltAt.IsZero() { + return err + } + rv.arm(ctx, cur) + return nil +} + +func (rv *reservations) arm(ctx context.Context, v versions.ReservedVersion) { + kv := rv.bounds() + var err error + if kv == nil { + err = fmt.Errorf("%s is not on the bus yet", versions.BoundsBucket) + } else if _, err = kv.Create(ctx, versions.ReservedBoundKey(v.Module, v.Series), versions.EncodeBound("the release of "+v.Module+" "+v.Version, time.Now().Add(rv.bound)), jetstream.KeyTTL(rv.bound)); errors.Is(err, jetstream.ErrKeyExists) { + err = nil + } + if err != nil { + rv.logf("the release bound of %s %s is not written, so it is raised unreleased now: %v", v.Module, v.Version, err) + rv.unreleased(ctx, v, fmt.Sprintf("its bound could not be written (%v)", err)) + } +} + +// unreleased raises version-unreleased about the module, naming the version and the artifacts missing. +func (rv *reservations) unreleased(ctx context.Context, v versions.ReservedVersion, why string) { + missing := v.Missing() + what := "an artifact is unpublished" + if len(missing) > 0 { + what = strings.Join(missing, ", ") + " unpublished" + } + rv.found.raise(ctx, finding{kind: versions.VersionUnreleased, scope: meshconditionsclient.ScopeModule, id: v.Module, + summary: fmt.Sprintf("%s %s is built and not released: %s (%s)", v.Module, v.Version, what, why), + headline: "A built version is not released", + explanation: "The builder reserved " + v.Module + " " + v.Version + " and its build ended, and the catalog has not said version-released: " + what + ".", + resolved: "The catalog says version-released for " + v.Module + " " + v.Version + ".", + needs: "builder_reserved shows the reservation; builder_publish_again publishes the missing artifacts under it; builder_reserved_drop drops it."}) +} + +// ended is told each change of the state bounds: a reservation's bound whose time ended raises version-unreleased while +// the reservation stands built. +func (rv *reservations) ended(ctx context.Context, e jetstream.KeyValueEntry) bool { + key, ok := versions.ReservedBoundOf(e.Key()) + if !ok { + return false + } + if e.Operation() != jetstream.KeyValuePurge { + return true + } + v, found, err := rv.r.At(ctx, key) + if err != nil || !found || v.BuiltAt.IsZero() { + return true + } + rv.unreleased(ctx, v, fmt.Sprintf("nothing within %s of its build", rv.bound)) + return true +} + +// dropBound ends a reservation's bound. +func (rv *reservations) dropBound(ctx context.Context, key string) { + if kv := rv.bounds(); kv != nil { + if err := kv.Delete(ctx, versions.ReservedBoundFor(key)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + rv.logf("the release bound of %s is not ended: %v", key, err) + } + } +} + +// heard takes the catalog's version-released and versions-filled in, and a registry seat's published (published): a +// message that is none of them is left to the others. A state that cannot take one leaves it for the bus to give again. +func (rv *reservations) heard(ctx context.Context, m jetstream.Msg) bool { + switch m.Subject() { + case meshcatalog.SubjectVersionsFilled: + rv.filled.Store(true) + rv.logf("the catalog said versions-filled: versions are reserved from now on") + if rv.wake != nil { + rv.wake(ctx) + } + case meshcatalog.SubjectVersionReleased: + var ev meshcatalog.VersionReleasedEvent + if err := json.Unmarshal(m.Data(), &ev); err != nil || ev.Module == "" || ev.Version == "" { + rv.logf("%s holds what is no version-released, left: %v", m.Subject(), err) + break + } + gone, err := rv.r.Released(ctx, ev.Module, ev.Version) + if err != nil { + rv.logf("%v; heard again", err) + if nakErr := m.Nak(); nakErr != nil { + rv.logf("leaving %s for the bus to give again: %v", m.Subject(), nakErr) + } + return true + } + for _, k := range gone { + rv.dropBound(ctx, k) + } + rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, ev.Module) + default: + return false + } + if err := m.Ack(); err != nil { + rv.logf("acknowledging %s: %v", m.Subject(), err) + } + return true +} + +// published marks the artifacts of the reservation a registry seat's published names (its version and digest). +func (rv *reservations) published(ctx context.Context, o versions.Outcome) { + if _, err := rv.r.Published(ctx, o.Version, o.Digest); err != nil { + rv.logf("a published of %s@%s is not recorded in the reservations: %v", o.Name, o.Version, err) + } +} + +// dropped is builder_reserved_drop: the reservation is deleted and its bound ended. +func (rv *reservations) dropped(ctx context.Context, module, series, by, why string, at time.Time) (string, error) { + ok, err := rv.r.Drop(ctx, module, series) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("%s holds no reservation of %s", module, series) + } + rv.dropBound(ctx, versions.ReservedKey(module, series)) + rv.found.clear(ctx, versions.VersionUnreleased, meshconditionsclient.ScopeModule, module) + return fmt.Sprintf("the reservation of %s %s is dropped (%s, %s)", module, series, by, why), nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go new file mode 100644 index 00000000..d1a04e2f --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reserve_test.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + + "mesh-delivery-builder/internal/versions" +) + +// memReserve is the state reserved in memory. +type memReserve struct { + vals map[string][]byte + revs map[string]uint64 +} + +func (m *memReserve) Get(_ context.Context, key string) ([]byte, uint64, bool, error) { + v, ok := m.vals[key] + return v, m.revs[key], ok, nil +} +func (m *memReserve) Create(_ context.Context, key string, v []byte) (uint64, error) { + if _, ok := m.vals[key]; ok { + return 0, versions.ErrConflict + } + m.vals[key], m.revs[key] = v, 1 + return 1, nil +} +func (m *memReserve) Update(_ context.Context, key string, v []byte, rev uint64) (uint64, error) { + if m.revs[key] != rev { + return 0, versions.ErrConflict + } + m.vals[key], m.revs[key] = v, rev+1 + return rev + 1, nil +} +func (m *memReserve) KeysOf(_ context.Context, filter string) ([]string, error) { + var out []string + for k := range m.vals { + if filter == versions.AllFilter || strings.HasPrefix(k, strings.TrimSuffix(filter, "*")) { + out = append(out, k) + } + } + return out, nil +} +func (m *memReserve) Delete(_ context.Context, key string) error { + delete(m.vals, key) + return nil +} + +// boundsKV keeps the keys created and deleted in the state bounds. +type boundsKV struct { + jetstream.KeyValue + keys map[string]bool +} + +func (b *boundsKV) Create(_ context.Context, key string, _ []byte, _ ...jetstream.KVCreateOpt) (uint64, error) { + b.keys[key] = true + return 1, nil +} +func (b *boundsKV) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error { + delete(b.keys, key) + return nil +} + +// TestTheCatalogsWordsMoveTheReservations: design 47 G §369, §372 (test 102) — versions-filled opens the reserving and wakes +// the builds that wait; a built reservation arms its bound and its end raises version-unreleased naming the artifacts +// missing; the catalog's version-released deletes the key and the bound and clears the finding; builder_reserved_drop drops one. +func TestTheCatalogsWordsMoveTheReservations(t *testing.T) { + ctx := context.Background() + st := &memFindings{vals: map[string][]byte{}} + found := &findings{store: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }} + bounds := &boundsKV{keys: map[string]bool{}} + woke := 0 + call := func(addr string, body any) (json.RawMessage, error) { + if addr != "seat:mesh-catalog.versions" { + t.Errorf("the released versions were asked of %q", addr) + } + return json.Marshal(meshcatalog.VersionsAnswer{Module: "m", Versions: []meshcatalog.VersionEntry{{Module: "m", Version: "1.4.2"}}}) + } + rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, call, func() jetstream.KeyValue { return bounds }, found, + func(context.Context) { woke++ }, t.Logf) + if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}); err == nil || rv.Filled() { + t.Fatalf("a version was reserved before versions-filled (%v)", err) + } + filled := &wakeMsg{subject: meshcatalog.SubjectVersionsFilled, data: []byte(`{"v":1}`)} + if !rv.heard(ctx, filled) || !filled.acked || !rv.Filled() || woke != 1 { + t.Fatalf("versions-filled: acked %v, filled %v, woke %d", filled.acked, rv.Filled(), woke) + } + res, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}) + if err != nil || res.Version != "1.4.3" { + t.Fatalf("reserved %q (%v), want 1.4.3 after the catalog's 1.4.2", res.Version, err) + } + if err := rv.Built(ctx, "m", "1.4.3", map[string]string{"m:a": "sha256:a", "m:b": "sha256:b"}); err != nil { + t.Fatal(err) + } + boundKey := versions.ReservedBoundKey("m", "1.4") + if !bounds.keys[boundKey] { + t.Fatalf("the publish's bound is not armed: %v", bounds.keys) + } + rv.published(ctx, versions.Outcome{Name: "m-a", Version: "1.4.3", Digest: "sha256:a"}) + if !rv.ended(ctx, purged{key: boundKey}) { + t.Fatal("a reservation's bound was not taken in") + } + key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m", versions.VersionUnreleased) + got, err := meshconditionsclient.DecodeFinding(st.vals[key]) + if err != nil || !strings.Contains(got.Summary, "m:b unpublished") || strings.Contains(got.Summary, "m:a unpublished") || !strings.Contains(got.Summary, "1.4.3") { + t.Fatalf("finding = %+v (%v), want version-unreleased naming m:b of 1.4.3", got, err) + } + released := &wakeMsg{subject: meshcatalog.SubjectVersionReleased, data: []byte(`{"v":1,"module":"m","version":"1.4.3","source":1,"artifacts":[]}`)} + if !rv.heard(ctx, released) || !released.acked { + t.Fatalf("version-released was not taken in (acked %v)", released.acked) + } + if all, _ := rv.r.All(ctx); len(all) != 0 || bounds.keys[boundKey] || len(st.vals) != 0 { + t.Errorf("after version-released: reservations %v, bounds %v, findings %v; want none", all, bounds.keys, st.vals) + } + if _, err := rv.Reserve(ctx, "m", "1.4", []string{"m:a"}); err != nil { + t.Fatal(err) + } + if answer, err := rv.dropped(ctx, "m", "1.4", "op", "never released", time.Unix(0, 0)); err != nil || !strings.Contains(answer, "dropped") { + t.Errorf("builder_reserved_drop: %q (%v)", answer, err) + } + if _, err := rv.dropped(ctx, "m", "1.4", "op", "again", time.Unix(0, 0)); err == nil { + t.Error("a reservation that is gone was dropped again") + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go index d6029f81..e3bff8b8 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/seats.go @@ -50,6 +50,8 @@ type seatHearer struct { logf func(string, ...any) // pub raises and clears the findings of a publish that ended badly; nil raises none. pub *publishes + // res marks a reservation's artifacts published; nil marks none. + res *reservations } // heard takes a message of the events consumer in when it is a registry seat's outcome, and answers whether it was. @@ -90,6 +92,9 @@ func (h *seatHearer) heard(ctx context.Context, m jetstream.Msg) bool { if h.pub != nil { h.pub.heard(ctx, se.seat, se.event, o) } + if h.res != nil && se.event == versions.HeardPublished { + h.res.published(ctx, o) + } h.ack(m) if len(wake) > 0 { h.logf("the floor of %s is asked again: %s said %s", strings.Join(wake, ", "), se.seat, se.event) diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go index 40d50d4e..8fa2946d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/versions.go @@ -29,7 +29,11 @@ const ( // awaitedVersions is the state versions as versions.Store: bound once the bus holds it (the grants part makes it, issue // 1790); until then every use is an error, so a trunk build is built unnumbered and said (build.Builder.number). -type awaitedVersions struct{ b *process.AwaitedBucket } +type awaitedVersions struct { + b *process.AwaitedBucket + // bucket names the state in the error of one not on the bus yet; empty is the versions'. + bucket string +} var errVersionsMissing = errors.New("the state " + versions.Bucket + " is not on the bus yet") @@ -37,6 +41,9 @@ func (s awaitedVersions) kv() (jetstream.KeyValue, error) { if kv := s.b.KV(); kv != nil { return kv, nil } + if s.bucket != "" { + return nil, errors.New("the state " + s.bucket + " is not on the bus yet") + } return nil, errVersionsMissing } diff --git a/modules/mesh-delivery-builder/internal/build/numbers.go b/modules/mesh-delivery-builder/internal/build/numbers.go index 6cf6250f..f1b9cef5 100644 --- a/modules/mesh-delivery-builder/internal/build/numbers.go +++ b/modules/mesh-delivery-builder/internal/build/numbers.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "strings" "time" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" @@ -140,3 +141,59 @@ func (b *Builder) PackageNamed(ctx context.Context, module string) { b.consider(ctx, module, m) } } + +// Reservations reserves the version of a trunk build at its start and records each artifact's digest (design 47 G §369). +// Nil reserves nothing. +type Reservations interface { + // Filled says the catalog said versions-filled: nothing is reserved before it. + Filled() bool + // Reserve reserves module's next version in series for the artifacts (their references, :). + Reserve(ctx context.Context, module, series string, artifacts []string) (versions.ReservedVersion, error) + // Built records the digests of the artifacts built under the reserved version, by reference. + Built(ctx context.Context, module, version string, digests map[string]string) error +} + +// reserves says whether a look at a trunk build is one that reserves: the build has a series of the form MAJOR.MINOR +// (design 47 G §368) and artifacts, and its key is not done already. +func (b *Builder) reserves(ctx context.Context, module string, m meshcatalog.ModulesByModuleValue, plan Plan) bool { + if b.Reserved == nil || len(plan.Tasks) == 0 { + return false + } + if strings.Count(plan.Series, ".") != 1 { + return false + } + return b.doneDigests(ctx, module, m.Fp2) == nil +} + +// waitFill holds a build that would reserve until the catalog says versions-filled, saying wait-held naming the fill once +// (design 47 G §369, §372). It answers whether the build waits. +func (b *Builder) waitFill(module string, m meshcatalog.ModulesByModuleValue) bool { + if b.Reserved.Filled() { + return false + } + if _, said := b.fillWaits.LoadOrStore(module+"."+m.Fp2, true); !said { + line := module + " at " + m.Fp2 + " is not built yet: wait-held, the catalog has not said versions-filled, and nothing is reserved before it" + b.logf("%s", line) + if b.Log != nil { + b.Log(builds.BuildLogEntry{Event: builds.BuildSaidEvent, Module: module, FP: m.Fp2, Commit: m.Commit, Node: b.Node, + At: time.Now().UTC(), Line: line, Why: "wait-held"}) + } + } + return true +} + +// reserve reserves the version of the build that starts; a reservation that cannot be made is said and the build goes on +// unreserved, as an unnumbered build does. +func (b *Builder) reserve(ctx context.Context, module string, plan Plan) (versions.ReservedVersion, bool) { + refs := make([]string, 0, len(plan.Tasks)) + for _, t := range plan.Tasks { + refs = append(refs, module+":"+t.Artifact) + } + res, err := b.Reserved.Reserve(ctx, module, plan.Series, refs) + if err != nil { + b.logf("%s: no version is reserved for this build: %v", module, err) + return versions.ReservedVersion{}, false + } + b.logf("%s: version %s is reserved for this build", module, res.Version) + return res, true +} diff --git a/modules/mesh-delivery-builder/internal/build/plan.go b/modules/mesh-delivery-builder/internal/build/plan.go index 11c678d5..bb9b1999 100644 --- a/modules/mesh-delivery-builder/internal/build/plan.go +++ b/modules/mesh-delivery-builder/internal/build/plan.go @@ -35,7 +35,11 @@ type Task struct { } // Plan is what a manifest asks the builder to build, by artifact name. -type Plan struct{ Tasks []Task } +type Plan struct { + Tasks []Task + // Series is the manifest's MAJOR.MINOR (design 47 G §368): the series its build reserves a version in (§369). + Series string +} // Modules are the build modules the plan calls, sorted. func (p Plan) Modules() []string { @@ -89,7 +93,7 @@ func PlanOf(m catalogue.Manifest) (Plan, error) { } artifacts := append([]catalogue.Artifact(nil), m.Build.Artifacts...) sort.Slice(artifacts, func(i, j int) bool { return artifacts[i].Name < artifacts[j].Name }) - var p Plan + p := Plan{Series: m.Version} for _, a := range artifacts { switch { case a.Kind == catalogue.ArtifactArchive: diff --git a/modules/mesh-delivery-builder/internal/build/reserve_test.go b/modules/mesh-delivery-builder/internal/build/reserve_test.go new file mode 100644 index 00000000..cc1fc2dd --- /dev/null +++ b/modules/mesh-delivery-builder/internal/build/reserve_test.go @@ -0,0 +1,104 @@ +package build + +import ( + "context" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "mesh-delivery-builder/internal/versions" +) + +// fakeReservations answers a fill and keeps what the builder reserved and recorded. +type fakeReservations struct { + filled bool + reserved [][]string + series []string + built map[string]string +} + +func (f *fakeReservations) Filled() bool { return f.filled } + +func (f *fakeReservations) Reserve(_ context.Context, module, series string, artifacts []string) (versions.ReservedVersion, error) { + f.reserved, f.series = append(f.reserved, artifacts), append(f.series, series) + return versions.ReservedVersion{Module: module, Series: series, Version: series + ".5"}, nil +} + +func (f *fakeReservations) Built(_ context.Context, _, version string, digests map[string]string) error { + f.built = map[string]string{version: ""} + for ref, d := range digests { + f.built[ref] = d + } + return nil +} + +// TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled: design 47 G §369, §372 (test 102) — nothing is reserved before +// versions-filled and the build waits saying wait-held naming the fill; after it the build reserves one version in the +// manifest's MAJOR.MINOR for its artifacts and records their digests; a done key reserves nothing again; a manifest whose +// version is no MAJOR.MINOR reserves nothing. +func TestABuildReservesItsVersionAtItsStartOnceTheCatalogFilled(t *testing.T) { + r := newRig(t, goBundle) + r.mf.byDigest[mdigest('a')] = catalogue.Manifest{Module: "m", Version: "0.1", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle}}} + res := &fakeReservations{} + r.b.Reserved = res + var said []builds.BuildLogEntry + r.b.Log = func(e builds.BuildLogEntry) { said = append(said, e) } + + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + r.b.Wait() + if _, ok := r.st.vals[r.key]; ok || len(res.reserved) != 0 { + t.Fatalf("before versions-filled the build was claimed (%v) or reserved %v", ok, res.reserved) + } + var waits []string + for _, e := range said { + if e.Event == builds.BuildSaidEvent && e.Why == "wait-held" { + waits = append(waits, e.Line) + } + } + if len(waits) != 1 || !slices.ContainsFunc(waits, func(l string) bool { return strings.Contains(l, "versions-filled") }) { + t.Fatalf("wait-held lines = %q, want one naming versions-filled", waits) + } + + res.filled = true + v := r.consider(t, "modules/m") + if v.State != builds.BuildDone || len(res.reserved) != 1 || !slices.Equal(res.reserved[0], []string{"m:tool"}) || res.series[0] != "0.1" { + t.Fatalf("build %+v reserved %v in %v, want done and one reservation of m:tool in 0.1", v.State, res.reserved, res.series) + } + if res.built["m:tool"] != digest || len(res.built) != 2 { + t.Errorf("recorded %v, want the digest of m:tool under the reserved version", res.built) + } + r.consider(t, "modules/m") + if len(res.reserved) != 1 { + t.Errorf("a done key reserved again: %v", res.reserved) + } + + // A manifest whose version is no MAJOR.MINOR reserves nothing and is built as before. + r2 := newRig(t, goBundle) + res2 := &fakeReservations{filled: true} + r2.b.Reserved = res2 + if v := r2.consider(t, "modules/m"); v.State != builds.BuildDone || len(res2.reserved) != 0 { + t.Errorf("a manifest with no MAJOR.MINOR: state %s, reserved %v", v.State, res2.reserved) + } +} + +// TestADropOfAReservationIsAReasksValueOfItsOwnKey: design 47 G §369 — builder_reserved_drop asks the builder through +// reasks, under a key of the reservation (never a build key), refusing a module with no MAJOR.MINOR or no why. +func TestADropOfAReservationIsAReasksValueOfItsOwnKey(t *testing.T) { + at := time.Unix(0, 0) + key, data, err := NewReservedDrop("m", "1.4", "never released", "op@x", at) + if err != nil || key != "reserved."+versions.ReservedKey("m", "1.4") { + t.Fatalf("key %q (%v)", key, err) + } + if r, err := DecodeReAsk(data); err != nil || r.Mode != ModeReservedDrop || r.Name != "m" || r.FP != "1.4" || r.By != "op@x" { + t.Errorf("value = %+v (%v)", r, err) + } + for name, args := range map[string][3]string{"no module": {"", "1.4", "why"}, "no series": {"m", "1", "why"}, "three numbers": {"m", "1.4.2", "why"}, "no why": {"m", "1.4", " "}} { + if _, _, err := NewReservedDrop(args[0], args[1], args[2], "op", at); err == nil { + t.Errorf("%s: a drop was made", name) + } + } +} diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index a7711f09..f5f8bf3a 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -95,6 +95,8 @@ type Builder struct { // Numbers numbers each trunk build at its claim and records what it was built with (design 51 *The versions*); nil // numbers nothing. Numbers Numbers + // Reserved reserves the version of a trunk build at its start (design 47 G §369); nil reserves nothing. + Reserved Reservations // Tags answers a check's build's tag from the kept tag asks; nil numbers no check's build. Tags TagOf // Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine @@ -103,16 +105,18 @@ type Builder struct { // its single reference, and its needs. Nil: no node states anything, and the first forms are kept. Forms func() (present, counted map[string][]string) - once sync.Once - sem chan struct{} - wg sync.WaitGroup - mu sync.Mutex - known map[string]meshcatalog.ModulesByModuleValue // by modules key - inflight map[string]bool // by build key - again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built - plans map[string]planned // by manifest digest - cands map[string]meshcatalog.CandidatesByGroupValue // by candidates key (candidates.go) - via map[string]via // by build key: a candidate's fetch (candidates.go) + once sync.Once + sem chan struct{} + wg sync.WaitGroup + mu sync.Mutex + // fillWaits are the builds that said wait-held for the fill (module.fingerprint), so it is said once. + fillWaits sync.Map + known map[string]meshcatalog.ModulesByModuleValue // by modules key + inflight map[string]bool // by build key + again map[string]meshcatalog.ModulesByModuleValue // by build key: the newest value that arrived while the key was being built + plans map[string]planned // by manifest digest + cands map[string]meshcatalog.CandidatesByGroupValue // by candidates key (candidates.go) + via map[string]via // by build key: a candidate's fetch (candidates.go) // outOfStep are, by build key, the out-of-step files a build's lanes answered, until its end entry takes them. outOfStep map[string][]string trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint @@ -491,6 +495,10 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB return } } + reserving := trunk && tag == "" && planErr == nil && !nothing && b.reserves(ctx, module, m, plan) + if reserving && b.waitFill(module, m) { + return + } // A key built again by hand is taken over as it stands too: one left building by a look that ended without its write // (issue 1966) is the hand tool's to finish. cl, err := b.Claimer.Claim(ctx, module, m.Fp2, m.Manifest, ClaimOptions{TakeOver: first || byHand, ByHand: byHand, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished, @@ -525,6 +533,11 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB } } } + var reserved versions.ReservedVersion + hasReserved := false + if reserving && cl.Build.State != builds.BuildDone { + reserved, hasReserved = b.reserve(ctx, module, plan) + } var result builds.BuilderBuild if nothing { // A manifest with no artifact is built by building nothing: its key is done with no artifact, claimed and written as @@ -580,6 +593,15 @@ func (b *Builder) try(ctx context.Context, module string, m meshcatalog.ModulesB b.logf("%s: %v", cl.Key, err) } } + if result.State == builds.BuildDone && hasReserved { + digests := map[string]string{} + for _, a := range result.Artifacts { + digests[module+":"+a.Name] = a.Digest + } + if err := b.Reserved.Built(ctx, module, reserved.Version, digests); err != nil { + b.logf("%s: the digests of %s are not recorded: %v", cl.Key, reserved.Version, err) + } + } if result.State == builds.BuildFailed { b.logf("%s: failed: %s", cl.Key, result.Error) } diff --git a/modules/mesh-delivery-builder/internal/build/stuck.go b/modules/mesh-delivery-builder/internal/build/stuck.go index e5ef2e86..77860c0f 100644 --- a/modules/mesh-delivery-builder/internal/build/stuck.go +++ b/modules/mesh-delivery-builder/internal/build/stuck.go @@ -120,6 +120,27 @@ func NewOldFormDrop(node, why, by string, at time.Time) (string, []byte, error) return "old-form." + node, data, err } +// ModeReservedDrop is the Mode of a reasks value of builder_reserved_drop (design 47 G section 369): a module's reservation +// of one MAJOR.MINOR dropped by hand; Name is the module and FP the MAJOR.MINOR. +const ModeReservedDrop = "reserved-drop" + +// NewReservedDrop is the value builder_reserved_drop writes: the key `reserved..` (never a build key) +// and the value. +func NewReservedDrop(module, series, why, by string, at time.Time) (string, []byte, error) { + module, series = strings.TrimSpace(module), strings.TrimSpace(series) + if module == "" { + return "", nil, errors.New("module is required") + } + if strings.Count(series, ".") != 1 { + return "", nil, fmt.Errorf("series %q is not MAJOR.MINOR", series) + } + if strings.TrimSpace(why) == "" { + return "", nil, errors.New("say why the reservation is dropped (why)") + } + data, err := json.Marshal(ReAsk{V: 1, Why: strings.TrimSpace(why), By: by, At: at.UTC(), Mode: ModeReservedDrop, Name: module, FP: series}) + return "reserved." + versions.ReservedKey(module, series), data, err +} + // ModeBuild is the Mode of a reasks value that asks the key to be built again, not its waiting asks to be asked again. const ModeBuild = "build" diff --git a/modules/mesh-delivery-builder/internal/versions/again.go b/modules/mesh-delivery-builder/internal/versions/again.go index 9e77235b..2f3e814d 100644 --- a/modules/mesh-delivery-builder/internal/versions/again.go +++ b/modules/mesh-delivery-builder/internal/versions/again.go @@ -217,6 +217,8 @@ type Acts struct { Remove func(ctx context.Context, seat string, e Entry) (RemoveOutcome, error) // DropOldForm drops a retired node from the old form's count by hand (builder_old_form): its answer, or why not. DropOldForm func(ctx context.Context, node, by, why string, at time.Time) (string, error) + // DropReserved drops a module's reservation of one MAJOR.MINOR by hand (builder_reserved_drop): its answer, or why not. + DropReserved func(ctx context.Context, module, series, by, why string, at time.Time) (string, error) } // TagAsked answers one unanswered value of tag-asks. An untag calls remove on each of the branch's tagged versions' diff --git a/modules/mesh-delivery-builder/internal/versions/reserve.go b/modules/mesh-delivery-builder/internal/versions/reserve.go new file mode 100644 index 00000000..36b02425 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/reserve.go @@ -0,0 +1,450 @@ +package versions + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" +) + +// **The reservation is one key per module and MAJOR.MINOR** (design 47 G §369, §52): the builder keeps in its old state +// section `reserved` the version its build reserved and each artifact's digest once built, written by compare-and-set at +// the build's start. A newer build of the module takes the next after the higher of the highest released and the one +// reserved, so two builds never share a version, and a superseded build's version is never released (a gap in the +// patches). The key is deleted when the catalog says `version-released` for it, or when a version of another MAJOR.MINOR +// of the module is released (the manifest moved on). This is no list of released versions: numbering reads those only from +// the catalog (§348), handed to Reserve by the caller. + +const ( + // ReservedState is the builder's old state section of the reservations. + ReservedState = "reserved" + // ReservedBucket is its bucket. + ReservedBucket = "mesh-delivery-builder_" + ReservedState + // ReservedTool is builder_reserved, ReservedDropTool builder_reserved_drop. + ReservedTool = "builder_reserved" + ReservedDropTool = "builder_reserved_drop" + // UnreleasedBound is the publish's bound: past it a built reservation with an artifact unpublished is the finding + // version-unreleased (§369). + UnreleasedBound = 10 * time.Minute + // VersionUnreleased is that finding. + VersionUnreleased = "version-unreleased" +) + +// ReservedVersion is the value of a reservation key. +type ReservedVersion struct { + V int `json:"v"` + Module string `json:"module"` + Series string `json:"series"` // MAJOR.MINOR + Version string `json:"version"` // MAJOR.MINOR.PATCH + // Artifacts are the module's artifacts at the version: the digest once built, empty before. + Artifacts map[string]string `json:"artifacts"` + // BuiltAt is when the build ended with every digest known; the publish's bound counts from it. + BuiltAt time.Time `json:"built-at,omitzero"` + // Published are the artifacts a registry seat said published at this version with the built digest. + Published map[string]bool `json:"published,omitempty"` + At time.Time `json:"at"` +} + +// ReservedKey is the key of module's reservation of series (MAJOR.MINOR). +func ReservedKey(module, series string) string { return Key(module, series) } + +// ErrNotFilled is what Reserve answers before the catalog says versions-filled: nothing is reserved until then (§369). +var ErrNotFilled = errors.New("versions-filled is not heard: nothing is reserved before the catalog's versions are filled") + +// ReserveStore is the reservations' bucket: compare-and-set, and deletion. +type ReserveStore interface { + Store + Delete(ctx context.Context, key string) error +} + +// Reserver reserves versions. +type Reserver struct { + Store ReserveStore + // ReleasedOf are the patch versions (MAJOR.MINOR.PATCH) the catalog holds released for module, from its verb versions. + ReleasedOf func(ctx context.Context, module string) ([]string, error) + // Filled says the catalog said versions-filled. + Filled func() bool + Now func() time.Time +} + +func (r *Reserver) now() time.Time { + if r.Now != nil { + return r.Now().UTC() + } + return time.Now().UTC() +} + +// SplitVersion is a MAJOR.MINOR.PATCH version's series and patch. +func SplitVersion(version string) (series string, patch int, err error) { + i := strings.LastIndex(version, ".") + if i < 0 || strings.Count(version, ".") != 2 { + return "", 0, fmt.Errorf("%q is not MAJOR.MINOR.PATCH", version) + } + patch, err = strconv.Atoi(version[i+1:]) + if err != nil || patch < 0 { + return "", 0, fmt.Errorf("%q is not MAJOR.MINOR.PATCH", version) + } + return version[:i], patch, nil +} + +// Reserve reserves the next version of module's series for a build that starts, for artifacts (their names), and answers +// it. The next patch is the one after the higher of the highest released and the one reserved, 0 when there is neither. A +// conflicting write (another build reserved first) is read again and the next taken. +func (r *Reserver) Reserve(ctx context.Context, module, series string, artifacts []string) (ReservedVersion, error) { + if r.Filled != nil && !r.Filled() { + return ReservedVersion{}, ErrNotFilled + } + released, err := r.ReleasedOf(ctx, module) + if err != nil { + return ReservedVersion{}, fmt.Errorf("the released versions of %s cannot be read: %w", module, err) + } + high := -1 + for _, v := range released { + if s, p, err := SplitVersion(v); err == nil && s == series && p > high { + high = p + } + } + key := ReservedKey(module, series) + for { + cur, rev, found, err := r.get(ctx, key) + if err != nil { + return ReservedVersion{}, err + } + next := high + if found { + if _, p, err := SplitVersion(cur.Version); err == nil && p > next { + next = p + } + } + out := ReservedVersion{V: 1, Module: module, Series: series, Version: fmt.Sprintf("%s.%d", series, next+1), Artifacts: map[string]string{}, At: r.now()} + for _, a := range artifacts { + out.Artifacts[a] = "" + } + raw, _ := json.Marshal(out) + if found { + _, err = r.Store.Update(ctx, key, raw, rev) + } else { + _, err = r.Store.Create(ctx, key, raw) + } + if errors.Is(err, ErrConflict) { + continue + } + if err != nil { + return ReservedVersion{}, fmt.Errorf("%s cannot be reserved: %w", key, err) + } + return out, nil + } +} + +func (r *Reserver) get(ctx context.Context, key string) (ReservedVersion, uint64, bool, error) { + raw, rev, found, err := r.Store.Get(ctx, key) + if err != nil || !found { + return ReservedVersion{}, 0, false, err + } + var v ReservedVersion + if err := json.Unmarshal(raw, &v); err != nil { + return ReservedVersion{}, 0, false, fmt.Errorf("%s holds what is not a reservation: %w", key, err) + } + return v, rev, true, nil +} + +// Built records artifact's digest under the reserved version of module's series; once every artifact has one the build is +// ended (BuiltAt). A reservation moved on to another version (a newer build took the next) is left alone and answers false. +func (r *Reserver) Built(ctx context.Context, module, version, artifact, digest string) (bool, error) { + series, _, err := SplitVersion(version) + if err != nil { + return false, err + } + key := ReservedKey(module, series) + for { + cur, rev, found, err := r.get(ctx, key) + if err != nil || !found || cur.Version != version { + return false, err + } + cur.Artifacts[artifact] = digest + if cur.BuiltAt.IsZero() && allSet(cur.Artifacts) { + cur.BuiltAt = r.now() + } + raw, _ := json.Marshal(cur) + if _, err := r.Store.Update(ctx, key, raw, rev); errors.Is(err, ErrConflict) { + continue + } else if err != nil { + return false, fmt.Errorf("%s cannot be written: %w", key, err) + } + return true, nil + } +} + +func allSet(m map[string]string) bool { + for _, d := range m { + if d == "" { + return false + } + } + return true +} + +// Released takes the catalog's version-released in: the key of that version is deleted, and every key of module's other +// series (the manifest moved on). A key holding a newer reserved version of the same series stays. It answers the keys +// deleted. +func (r *Reserver) Released(ctx context.Context, module, version string) ([]string, error) { + series, _, err := SplitVersion(version) + if err != nil { + return nil, err + } + keys, err := r.Store.KeysOf(ctx, NameFilter(module)) + if err != nil { + return nil, fmt.Errorf("the reservations of %s cannot be listed: %w", module, err) + } + var gone []string + for _, k := range keys { + cur, _, found, err := r.get(ctx, k) + if err != nil { + return gone, err + } + if !found || (cur.Series == series && cur.Version != version) { + continue + } + if err := r.Store.Delete(ctx, k); err != nil { + return gone, fmt.Errorf("%s cannot be deleted: %w", k, err) + } + gone = append(gone, k) + } + sort.Strings(gone) + return gone, nil +} + +// Unreleased are the reservations whose build ended more than UnreleasedBound before now with an artifact unpublished: +// the finding version-unreleased, naming the module, the version and the artifacts missing (published says whether an +// artifact is). A reservation whose build has not ended is not one. +type Unreleased struct { + Module, Version string + Missing []string +} + +// Overdue answers the reservations past the bound with an artifact not published. +func (r *Reserver) Overdue(ctx context.Context, published func(module, version, artifact string) bool) ([]Unreleased, error) { + keys, err := r.Store.KeysOf(ctx, AllFilter) + if err != nil { + return nil, fmt.Errorf("the reservations cannot be listed: %w", err) + } + sort.Strings(keys) + var out []Unreleased + for _, k := range keys { + cur, _, found, err := r.get(ctx, k) + if err != nil { + return nil, err + } + if !found || cur.BuiltAt.IsZero() || r.now().Sub(cur.BuiltAt) < UnreleasedBound { + continue + } + var missing []string + for _, a := range cur.Missing() { + if published == nil || !published(cur.Module, cur.Version, a) { + missing = append(missing, a) + } + } + if len(missing) > 0 { + out = append(out, Unreleased{Module: cur.Module, Version: cur.Version, Missing: missing}) + } + } + return out, nil +} + +// All are the reservations, sorted by key: builder_reserved. +func (r *Reserver) All(ctx context.Context) ([]ReservedVersion, error) { + keys, err := r.Store.KeysOf(ctx, AllFilter) + if err != nil { + return nil, fmt.Errorf("the reservations cannot be listed: %w", err) + } + sort.Strings(keys) + var out []ReservedVersion + for _, k := range keys { + cur, _, found, err := r.get(ctx, k) + if err != nil { + return nil, err + } + if found { + out = append(out, cur) + } + } + return out, nil +} + +// Drop deletes module's reservation of series on a person's word (builder_reserved_drop); the caller writes the hand-act +// log. It answers whether there was one. +func (r *Reserver) Drop(ctx context.Context, module, series string) (bool, error) { + key := ReservedKey(module, series) + _, _, found, err := r.get(ctx, key) + if err != nil || !found { + return false, err + } + return true, r.Store.Delete(ctx, key) +} + +// Published takes a registry seat's published in: the artifacts of a reservation at version (a leading v is the Go form) +// whose built digest it names are marked published. It answers the keys it marked. +func (r *Reserver) Published(ctx context.Context, version, digest string) ([]string, error) { + version = strings.TrimPrefix(version, "v") + if version == "" || digest == "" { + return nil, nil + } + keys, err := r.Store.KeysOf(ctx, AllFilter) + if err != nil { + return nil, fmt.Errorf("the reservations cannot be listed: %w", err) + } + sort.Strings(keys) + var marked []string + for _, k := range keys { + again: + cur, rev, found, err := r.get(ctx, k) + if err != nil { + return marked, err + } + if !found || cur.Version != version { + continue + } + changed := false + for a, d := range cur.Artifacts { + if d == digest && !cur.Published[a] { + if cur.Published == nil { + cur.Published = map[string]bool{} + } + cur.Published[a], changed = true, true + } + } + if !changed { + continue + } + raw, _ := json.Marshal(cur) + if _, err := r.Store.Update(ctx, k, raw, rev); errors.Is(err, ErrConflict) { + goto again + } else if err != nil { + return marked, fmt.Errorf("%s cannot be written: %w", k, err) + } + marked = append(marked, k) + } + return marked, nil +} + +// Of is module's reservation of series, and whether there is one; with an empty series the module's highest. +func (r *Reserver) Of(ctx context.Context, module, version string) (ReservedVersion, bool, error) { + all, err := r.All(ctx) + if err != nil { + return ReservedVersion{}, false, err + } + var out ReservedVersion + found := false + for _, v := range all { + if v.Module != module || (version != "" && v.Version != version) { + continue + } + if !found { + out, found = v, true + continue + } + _, p, _ := SplitVersion(v.Version) + _, q, _ := SplitVersion(out.Version) + if v.Series > out.Series || (v.Series == out.Series && p > q) { + out = v + } + } + return out, found, nil +} + +const reservedBoundPrefix = "reserved." + +// ReservedBoundKey is the bound of a reservation's publish in the state bounds (key-ttl): `reserved.`, armed +// when its build ended with every digest known. Its end while the reservation stands is the finding version-unreleased. +func ReservedBoundKey(module, series string) string { + return ReservedBoundFor(ReservedKey(module, series)) +} + +// ReservedBoundFor is the bound of the reservation under key. +func ReservedBoundFor(key string) string { return reservedBoundPrefix + key } + +// ReservedBoundOf is the reserved key a reservation's bound names, and whether key is one. +func ReservedBoundOf(key string) (string, bool) { return strings.CutPrefix(key, reservedBoundPrefix) } + +// readOnlyReserve is the reservations read by the tools bundle, which writes nothing. +type readOnlyReserve struct{ readStore } + +func (readOnlyReserve) Delete(context.Context, string) error { return errReadOnly } + +// AnswerReserved is builder_reserved: every reservation, or one module's. +func AnswerReserved(r Reader, module string) (map[string]any, error) { + rs := &Reserver{Store: readOnlyReserve{readStore{r}}} + all, err := rs.All(context.Background()) + if err != nil { + return nil, err + } + out := []ReservedVersion{} + for _, v := range all { + if module == "" || v.Module == module { + out = append(out, v) + } + } + return map[string]any{"reserved": out}, nil +} + +// AnswerReservedVerb is the seat mesh-builds' verb reserved: module's reserved version (the given one, else the highest) +// with each artifact's digest, an artifact not built yet with none. +func AnswerReservedVerb(r Reader, module, version string) (meshbuilds.ReservedAnswer, error) { + module = strings.TrimSpace(module) + if module == "" { + return meshbuilds.ReservedAnswer{}, errors.New("module is required") + } + rs := &Reserver{Store: readOnlyReserve{readStore{r}}} + v, found, err := rs.Of(context.Background(), module, strings.TrimSpace(version)) + if err != nil { + return meshbuilds.ReservedAnswer{}, err + } + if !found { + return meshbuilds.ReservedAnswer{}, fmt.Errorf("%s has no reserved version%s", module, map[bool]string{true: " " + version, false: ""}[version != ""]) + } + out := meshbuilds.ReservedAnswer{Module: v.Module, Version: v.Version, Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{}} + names := make([]string, 0, len(v.Artifacts)) + for a := range v.Artifacts { + names = append(names, a) + } + sort.Strings(names) + for _, a := range names { + out.Artifacts = append(out.Artifacts, meshbuilds.ReservedAnswerArtifactsItem{Artifact: a, Digest: v.Artifacts[a]}) + } + return out, nil +} + +// The descriptions of the reservation's tools. +const ( + ReservedDescription = "The versions the builder reserved (its state reserved, design 47 G section 369): one per module and MAJOR.MINOR, " + + "the version its build reserved at its start, each artifact's digest once built, when the build ended and which artifacts a registry " + + "seat said published. With module, that module's only. A key goes when the catalog says version-released for it. Read-only." + ReservedDropDescription = "Drop a module's reservation of one MAJOR.MINOR by hand (design 47 G section 369), when its version will never be " + + "released; the next build takes the next after the highest released. A hand act: it is recorded in the hand-act log." + // ReservedVerb is the seat mesh-builds' verb served as a tool of the same name. + ReservedVerb = meshbuilds.VerbReserved +) + +// Missing are the artifacts of v no registry seat said published, sorted. +func (v ReservedVersion) Missing() []string { + var out []string + for a := range v.Artifacts { + if !v.Published[a] { + out = append(out, a) + } + } + sort.Strings(out) + return out +} + +// At is the reservation under key, and whether there is one. +func (r *Reserver) At(ctx context.Context, key string) (ReservedVersion, bool, error) { + v, _, found, err := r.get(ctx, key) + return v, found, err +} diff --git a/modules/mesh-delivery-builder/internal/versions/reserve_test.go b/modules/mesh-delivery-builder/internal/versions/reserve_test.go new file mode 100644 index 00000000..c02c0c96 --- /dev/null +++ b/modules/mesh-delivery-builder/internal/versions/reserve_test.go @@ -0,0 +1,158 @@ +package versions + +import ( + "encoding/json" + + "context" + "errors" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" + "reflect" + "testing" + "time" +) + +type reserveMem struct { + *mem + onUpdate func() // runs once before the next Update +} + +func (m *reserveMem) Delete(_ context.Context, key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.vals, key) + delete(m.revs, key) + return nil +} + +func (m *reserveMem) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) { + if f := m.onUpdate; f != nil { + m.onUpdate = nil + f() + } + return m.mem.Update(ctx, key, value, rev) +} + +// TestReservation: design 47 G §369 (test 102): two builds close together reserve two versions, the key goes on +// version-released, a built reservation with an artifact unpublished past the bound is unreleased, and nothing is reserved +// before versions-filled. +func TestReservation(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC) + released := []string{"0.1.18", "0.1.19", "0.2.7"} + filled := true + store := &reserveMem{mem: newMem()} + r := &Reserver{Store: store, Filled: func() bool { return filled }, Now: func() time.Time { return now }, + ReleasedOf: func(context.Context, string) ([]string, error) { return released, nil }} + arts := []string{"m:a", "m:b"} + + filled = false + if _, err := r.Reserve(ctx, "m", "0.1", arts); !errors.Is(err, ErrNotFilled) { + t.Fatalf("before versions-filled: %v, want ErrNotFilled", err) + } + if got, _ := r.All(ctx); len(got) != 0 { + t.Fatalf("a reservation was written before the fill: %v", got) + } + filled = true + + first, err := r.Reserve(ctx, "m", "0.1", arts) + if err != nil || first.Version != "0.1.20" { + t.Fatalf("first build reserves %q (%v), want 0.1.20 after the highest released 0.1.19", first.Version, err) + } + second, err := r.Reserve(ctx, "m", "0.1", arts) + if err != nil || second.Version != "0.1.21" { + t.Fatalf("second build reserves %q (%v), want 0.1.21: never the first's", second.Version, err) + } + // Two builds at once: the rival reserves between this one's read and write, and this one takes the next. + store.onUpdate = func() { + if _, err := r.Reserve(ctx, "m", "0.1", arts); err != nil { + t.Error(err) + } + } + third, err := r.Reserve(ctx, "m", "0.1", arts) + if err != nil || third.Version != "0.1.23" { + t.Fatalf("a build that lost the compare-and-set reserves %q (%v), want 0.1.23", third.Version, err) + } + if v, _ := r.Reserve(ctx, "m", "0.3", arts); v.Version != "0.3.0" { + t.Errorf("a series with nothing released reserves %q, want 0.3.0", v.Version) + } + + // The artifacts' digests, then the bound. + for _, a := range arts { + if ok, err := r.Built(ctx, "m", "0.1.23", a, "sha256:"+a); err != nil || !ok { + t.Fatalf("built %s: %v %v", a, ok, err) + } + } + if ok, _ := r.Built(ctx, "m", "0.1.22", "m:a", "x"); ok { + t.Error("a digest was written under a version a newer build superseded") + } + published := func(_, _, a string) bool { return a == "m:a" } + if got, _ := r.Overdue(ctx, published); len(got) != 0 { + t.Fatalf("unreleased before the bound: %v", got) + } + now = now.Add(UnreleasedBound) + got, err := r.Overdue(ctx, published) + want := []Unreleased{{Module: "m", Version: "0.1.23", Missing: []string{"m:b"}}} + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("past the bound: %v (%v), want %v", got, err, want) + } + + // version-released: its key goes, and the other series' keys of the module (the manifest moved on). + if gone, err := r.Released(ctx, "m", "0.1.23"); err != nil || len(gone) != 2 { + t.Fatalf("released 0.1.23 deleted %v (%v), want the 0.1 key and the 0.3 key", gone, err) + } + if all, _ := r.All(ctx); len(all) != 0 { + t.Errorf("reservations left: %v", all) + } + if again, _ := r.Reserve(ctx, "m", "0.1", arts); again.Version != "0.1.20" { + t.Logf("with the catalog list unchanged the next is %s", again.Version) + } + if ok, _ := r.Drop(ctx, "m", "0.1"); !ok { + t.Error("builder_reserved_drop found no reservation") + } +} + +// TestReservationIsReadAndMarkedPublished: design 47 G §369 — the reservation is read by builder_reserved and by the seat's +// verb reserved with each artifact's digest, a registry seat's published marks the artifacts of that version and digest, and +// the missing artifacts it names are the unpublished ones. +func TestReservationIsReadAndMarkedPublished(t *testing.T) { + ctx := context.Background() + store := &reserveMem{mem: newMem()} + r := &Reserver{Store: store, ReleasedOf: func(context.Context, string) ([]string, error) { return []string{"1.4.2"}, nil }} + v, err := r.Reserve(ctx, "m", "1.4", []string{"m:a", "m:b"}) + if err != nil || v.Version != "1.4.3" { + t.Fatalf("reserved %q (%v), want 1.4.3", v.Version, err) + } + if _, err := r.Built(ctx, "m", "1.4.3", "m:a", "sha256:aa"); err != nil { + t.Fatal(err) + } + rd := Reader{ + Keys: func() ([]string, error) { return store.KeysOf(ctx, AllFilter) }, + Get: func(key string) (json.RawMessage, bool, error) { + raw, _, ok, err := store.Get(ctx, key) + return raw, ok, err + }} + ans, err := AnswerReservedVerb(rd, "m", "") + want := meshbuilds.ReservedAnswer{Module: "m", Version: "1.4.3", Artifacts: []meshbuilds.ReservedAnswerArtifactsItem{{Artifact: "m:a", Digest: "sha256:aa"}, {Artifact: "m:b"}}} + if err != nil || !reflect.DeepEqual(ans, want) { + t.Fatalf("reserved answers %+v (%v), want %+v", ans, err, want) + } + if _, err := AnswerReservedVerb(rd, "m", "1.4.9"); err == nil { + t.Error("a version never reserved was answered") + } + if got, err := AnswerReserved(rd, "other"); err != nil || len(got["reserved"].([]ReservedVersion)) != 0 { + t.Errorf("builder_reserved of another module = %v (%v), want none", got, err) + } + if marked, err := r.Published(ctx, "v1.4.3", "sha256:aa"); err != nil || len(marked) != 1 { + t.Fatalf("a published of 1.4.3 with a built digest marked %v (%v)", marked, err) + } + if marked, _ := r.Published(ctx, "1.4.3", "sha256:other"); len(marked) != 0 { + t.Errorf("a published with another digest marked %v", marked) + } + cur, _, _ := r.At(ctx, ReservedKey("m", "1.4")) + if got := cur.Missing(); !reflect.DeepEqual(got, []string{"m:b"}) { + t.Errorf("missing = %v, want m:b", got) + } + if key, ok := ReservedBoundOf(ReservedBoundKey("m", "1.4")); !ok || key != ReservedKey("m", "1.4") || ReservedBoundFor(key) != ReservedBoundKey("m", "1.4") { + t.Errorf("the bound key does not name the reservation back: %q %v", key, ok) + } +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index dae603fa..d03eb00c 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -38,9 +38,14 @@ type artifact struct { } type manifest struct { - Module string `json:"module"` - Version string `json:"version"` - Slug string `json:"slug"` + Module string `json:"module"` + Version string `json:"version"` + Slug string `json:"slug"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + Serves []string `json:"serves"` + } `json:"claims"` Tools []string `json:"tools"` Emits []string `json:"emits"` Consumes []string `json:"consumes"` @@ -97,7 +102,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}, {Name: "tag-ask-bounds", History: 1, KeyTTL: true}, {Name: "tag-asks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "old-form", History: 1}, {Name: "bounds", History: 1, KeyTTL: true}, {Name: "queues", History: 1}, {Name: "reserved", History: 1}, {Name: "reasks", History: 1}, {Name: "tag-ask-bounds", History: 1, KeyTTL: true}, {Name: "tag-asks", History: 1}, {Name: "versions", History: 1}, {Name: "first-views", History: 1}}; !reflect.DeepEqual(m.State, want) { // builds.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -127,7 +132,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // The watchdogs seat's presence says a node's machine is heard (issue 1966, ADR 0596 §2, ADR 0668 §3). if want := []string{"mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", - "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", + "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", "seat:node-build-image.drop", "seat:node-build-node.drop", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) } @@ -139,7 +144,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // DLV-M1: the indexer's and the assigner's views families of the states it follows. if want := []string{"container-image-registry.kept", "container-image-registry.published", "container-image-registry.refused", "container-image-registry.removed", "go-package-registry.kept", "go-package-registry.published", "go-package-registry.refused", "go-package-registry.removed", - "mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-catalog.manifest-accepted", + "mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-catalog.manifest-accepted", "mesh-catalog.version-released", "mesh-catalog.versions-filled", "mesh-conditions.condition-cleared", "mesh-delivery-assigner.views.assignments", "mesh-delivery-checker.tag-wanted", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.packages", "node-engine.manifest-reads-changed", @@ -155,7 +160,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. if want := []string{"builder_announce_again", "builder_ask_again", "builder_bounds", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_first_view_again", "builder_first_views", - "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_seat_queue", "builder_stuck", "builder_tag", "builder_tag_asks", "builder_untag", "builder_versions"}; !reflect.DeepEqual(sorted(m.Tools), want) { + "builder_floor_again", "builder_floor_waits", "builder_old_form", "builder_publish_again", "builder_reserved", "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_tag", "builder_tag_asks", "builder_untag", "builder_versions", "reserved"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // Design 51 *Events of the builder*: numbered and old-form-closed on the family version, apart from the builds' ends. @@ -166,7 +171,11 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if len(m.Logs) != 1 || m.Logs[0].Name != builds.BuilderLog || m.Logs[0].MaxAgeDays != builds.BuilderLogDays || m.Logs[0].MaxMiB != builds.BuilderLogMiB { t.Errorf("logs = %+v, want %s kept %d days in %d MiB", m.Logs, builds.BuilderLog, builds.BuilderLogDays, builds.BuilderLogMiB) } - for _, k := range []string{"streams", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { + // Design 47 G §52, §369: the seat mesh-builds, held once for the mesh, serving reserved. + if len(m.Claims) != 1 || m.Claims[0].Name != "mesh-builds" || m.Claims[0].Scope != "mesh" || !reflect.DeepEqual(m.Claims[0].Serves, []string{"reserved"}) { + t.Errorf("claims = %+v, want mesh-builds, scope mesh, serving reserved", m.Claims) + } + for _, k := range []string{"streams", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the builder has none", k) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index 9ab32e47..39a0cc6b 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -2,6 +2,15 @@ "module": "mesh-delivery-builder", "version": "1", "slug": "builder", + "claims": [ + { + "name": "mesh-builds", + "scope": "mesh", + "serves": [ + "reserved" + ] + } + ], "tools": [ "builder_announce_again", "builder_ask_again", @@ -13,12 +22,15 @@ "builder_floor_waits", "builder_old_form", "builder_publish_again", + "builder_reserved", + "builder_reserved_drop", "builder_seat_queue", "builder_stuck", "builder_tag", "builder_tag_asks", "builder_untag", "builder_versions", + "reserved", "builder_first_views", "builder_first_view_again" ], @@ -40,6 +52,8 @@ "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-catalog.manifest-accepted", + "mesh-catalog.version-released", + "mesh-catalog.versions-filled", "mesh-conditions.condition-cleared", "mesh-delivery-checker.tag-wanted", "mesh-delivery-forge-bridge.forge.**", @@ -66,6 +80,7 @@ "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", + "seat:mesh-catalog.versions", "seat:mesh-hand-acts.hand-act", "seat:mesh-watchdogs.presence", "seat:node-build-go.drop", @@ -92,6 +107,7 @@ { "name": "old-form", "history": 1 }, { "name": "bounds", "history": 1, "key-ttl": true }, { "name": "queues", "history": 1 }, + { "name": "reserved", "history": 1 }, { "name": "reasks", "history": 1 }, { "name": "tag-ask-bounds", "history": 1, "key-ttl": true }, { "name": "tag-asks", "history": 1 }, From d813ce7e3976b973f9d752d6d94da83db2bf1ef9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:45:19 +0100 Subject: [PATCH 04/16] FB4a: the forge message forms of an owed act's outcome (owed.ended) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate's developer needs OwedEnded and its outcomes; fields are the ones §164 and §168 name (id, outcome, words, the seat that said the act, time). Builds: design 47 G §164, §168 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../forgemsg/forge_test.go | 34 +++++++++ .../forgemsg/owed.go | 76 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 modules/mesh-delivery-forge-bridge/forgemsg/owed.go diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go index a94f4779..18c75b4c 100644 --- a/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go +++ b/modules/mesh-delivery-forge-bridge/forgemsg/forge_test.go @@ -5,6 +5,7 @@ import ( "errors" "strings" "testing" + "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-sdk/go/keys" @@ -191,4 +192,37 @@ func TestForge(t *testing.T) { } } }) + // Design 47 G §164: an owed act's outcome names its seat, its id, one outcome and words for every outcome but + // performed; an act the bridge cannot decode may name no id. + t.Run("owed ended", func(t *testing.T) { + at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) + ok := OwedEnded{Of: "mesh-delivery", ID: "g1.view.1", Outcome: OwedPerformed, At: at} + raw, err := EncodeOwedEnded(ok) + if back, derr := DecodeOwedEnded(raw); err != nil || derr != nil || back.V != Version || back.ID != ok.ID { + t.Fatalf("round trip: %s %v %v", raw, err, derr) + } + if _, err := DecodeOwedEnded([]byte(`{"v":99,"of":"x","id":"a","outcome":"performed","at":"2026-11-09T20:00:00Z"}`)); err == nil { + t.Errorf("an unknown version was read") + } + for _, c := range []struct { + name string + edit func(*OwedEnded) + want string + }{ + {"no seat", func(e *OwedEnded) { e.Of = "" }, "no seat"}, + {"an outcome of no kind", func(e *OwedEnded) { e.Outcome = "done" }, "none of"}, + {"no id", func(e *OwedEnded) { e.ID = "" }, "no id"}, + {"a refusal with no words", func(e *OwedEnded) { e.Outcome = OwedRefused }, "no words"}, + {"dropped with no reason", func(e *OwedEnded) { e.Outcome = OwedDropped }, "no words"}, + {"no time", func(e *OwedEnded) { e.At = time.Time{} }, "no time"}, + {"undecodable with no id is said", func(e *OwedEnded) { e.ID, e.Outcome, e.Words = "", OwedUndecodable, "not json" }, ""}, + } { + e := ok + c.edit(&e) + _, err := EncodeOwedEnded(e) + if (c.want == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), c.want)) { + t.Errorf("%s: %v, want %q", c.name, err, c.want) + } + } + }) } diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/owed.go b/modules/mesh-delivery-forge-bridge/forgemsg/owed.go new file mode 100644 index 00000000..b8400a36 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/forgemsg/owed.go @@ -0,0 +1,76 @@ +package forgemsg + +import ( + "encoding/json" + "fmt" + "slices" + "time" +) + +// OwedEndedEvent is the name of an owed act's outcome on the forge bridge's stream `forge` (design 47 G §164): the +// bridge says it for each act the gate or the checker said as the event `owed`. Its two parts are no repository's +// forge event (`forge...`). +const OwedEndedEvent = "owed.ended" + +// The outcomes of an owed act (design 47 G §164, §168): performed, refused (the forge's own words), could not be +// performed (the forge did not answer within the call's bound, with the error), undecodable (an act the bridge cannot +// read, with why) and, from the tool `bridge_owed_drop`, dropped. +const ( + OwedPerformed = "performed" + OwedRefused = "refused" + OwedCouldNotBePerformed = "could not be performed" + OwedUndecodable = "undecodable" + OwedDropped = "dropped" +) + +// OwedOutcomes are the outcomes an owed.ended says, one of them. +var OwedOutcomes = []string{OwedPerformed, OwedRefused, OwedCouldNotBePerformed, OwedUndecodable, OwedDropped} + +// OwedEnded is an owed act's outcome (design 47 G §164): the act's id and one outcome, with the words of any outcome +// but performed. Of is the seat that said the act (mesh-delivery for the gate's, mesh-delivery-checker for the +// checker's), so each hears its own ids (§164). An act the bridge cannot decode may name no id. +type OwedEnded struct { + V int `json:"v"` + Of string `json:"of"` + ID string `json:"id,omitempty"` + Outcome string `json:"outcome"` + Words string `json:"words,omitempty"` + At time.Time `json:"at"` +} + +func (e OwedEnded) check() error { + switch { + case e.Of == "": + return fmt.Errorf("an owed act's outcome names no seat that said it") + case !slices.Contains(OwedOutcomes, e.Outcome): + return fmt.Errorf("the outcome %q of an owed act is none of %v", e.Outcome, OwedOutcomes) + case e.ID == "" && e.Outcome != OwedUndecodable: + return fmt.Errorf("the outcome %s of an owed act names no id", e.Outcome) + case e.Outcome != OwedPerformed && e.Words == "": + return fmt.Errorf("the outcome %s of the owed act %s says no words", e.Outcome, e.ID) + case e.At.IsZero(): + return fmt.Errorf("the outcome of the owed act %s says no time", e.ID) + } + return nil +} + +// EncodeOwedEnded is e as JSON at Version; it refuses what check refuses. +func EncodeOwedEnded(e OwedEnded) ([]byte, error) { + if err := e.check(); err != nil { + return nil, err + } + e.V = Version + return json.Marshal(e) +} + +// DecodeOwedEnded reads an owed act's outcome; it refuses an unknown version and what EncodeOwedEnded refuses. +func DecodeOwedEnded(data []byte) (OwedEnded, error) { + var e OwedEnded + if err := decode(data, &e); err != nil { + return OwedEnded{}, err + } + if err := e.check(); err != nil { + return OwedEnded{}, err + } + return e, nil +} From 154505f451ae9da7f6a8ad2e2ee0f58031cca04d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:45:37 +0100 Subject: [PATCH 05/16] Assigner: repair against the new contracts (source key per branch, want with digests, seat verbs not yet served) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §15, §95, §353, §379 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/pass_test.go | 8 ++++---- .../cmd/mesh-delivery-assigner/wants_tools.go | 3 +++ .../cmd/mesh-delivery-assigner/world.go | 5 ++++- .../mesh-delivery-assigner/internal/assign/wants_test.go | 3 ++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 5b077f78..0bc60e9a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -1128,10 +1128,10 @@ func aReleasesApproveAnswersTheMoveOfItsBuildAlone(t *testing.T) { p.logf = t.Logf mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpTest, Repository: "novox/web", Commit: commit, Src: 7, FpSrc: c.fpSrc, Path: meshcatalogclient.RootPath}) p.w.put(buckets.Modules, "web", mod) - src, _ := mdc.EncodeSource(mdseat.SourceByMainValue{Group: "feat/web", Heads: map[string]string{"novox/web": commit}}) - p.w.putEntry(buckets.Source, mdc.SourceMainKey, src, 7, time.Now()) - later, _ := mdc.EncodeSource(mdseat.SourceByMainValue{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}}) - p.w.putEntry(buckets.Source, mdc.SourceMainKey, later, 8, time.Now()) // a later merge leaves revision 7's release + src, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/web", Heads: map[string]string{"novox/web": commit}}) + p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), src, 7, time.Now()) + later, _ := mdc.EncodeSource(mdseat.SourceByBranchValue{Group: "feat/other", Heads: map[string]string{"novox/web": commit, "novox/x": commit}}) + p.w.putEntry(buckets.Source, mdc.SourceKey(trunkBranch), later, 8, time.Now()) // a later merge leaves revision 7's release if c.mergedAt != "" { merge, err := mdc.EncodeMerge(mdseat.MergesByGroupValue{Group: "feat/web", Ask: "merge.feat_web.0123456789ab", Pulls: []mdseat.MergesByGroupValuePullsItem{ {Repository: "novox/web", Number: 3, Head: strings.Repeat("3", 40), State: string(mdc.MergeMerged), Commit: c.mergedAt}}}) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go index 02a0d387..034e9b15 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/wants_tools.go @@ -315,6 +315,9 @@ func (s *wantsTools) tools() []stdio.Tool { } var out []stdio.Tool for _, v := range meshwants.Verbs() { + if run[v] == nil { + continue // a verb of the seat this module does not serve yet: revision, entry, difference (AS11), input (AS5) + } out = append(out, stdio.Tool{Name: meshwants.Seat + "." + v, Run: run[v]}) } return out diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index cf2b67f4..893d02f5 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -86,7 +86,7 @@ func (w *world) putEntry(bucket, key string, value []byte, revision uint64, at t if bucket == plc.AssignmentsBucket { _, _ = w.assignmentKeys.Put(key, value, revision, false) // put took it already; this is its revision } - if bucket == buckets.Source && key == mdc.SourceMainKey { + if bucket == buckets.Source && key == mdc.SourceKey(trunkBranch) { if s, err := mdc.DecodeSource(value); err == nil && s.Group != "" { w.sourceGroups[revision] = s.Group } @@ -575,3 +575,6 @@ func (w *world) unplacedNodes() []string { sort.Strings(out) return out } + +// trunkBranch is the branch of the trunk-only want, kept until the first registrations (design 47 G §95, §353). +const trunkBranch = "main" diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index d15823b1..1c38e3a7 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -9,6 +9,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" "mesh-delivery-assigner/internal/puts" + "reflect" "slices" "strings" "testing" @@ -431,7 +432,7 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { before["g14"] = mwc.Want{Fp: fNew, Composed: dig(fNew)} } out := Wants(m, n, before, nil) - if got := out.Wants["g14"]; got != c.want { + if got := out.Wants["g14"]; !reflect.DeepEqual(got, c.want) { t.Errorf("%s: g14's want = %+v, want %+v", c.name, got, c.want) } if got := out.Rules["g14"].Rule; got != c.rule { From 18c954ddcd2d0d8167101a0e88e656a875675752 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:45:50 +0100 Subject: [PATCH 06/16] CT10: meshwantsclient holds a part's view of the assignment half, a first view then assignment-revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The typed first view and the following of each half sit in the seats' client packages, built on go/views (design 47 G §245), so no part follows the assignment half by a follower of its own: a newer trunk revision heard is waited for, said in the status, asked and held; a candidate's, or one held, asks nothing. Builds: design 47 G §245, §365 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../go/clients/meshwantsclient/assignment.go | 176 ++++++++++++++++++ .../meshwantsclient/assignment_test.go | 116 ++++++++++++ contracts/go/clients/meshwantsclient/go.mod | 9 + contracts/go/clients/meshwantsclient/go.sum | 24 +++ 4 files changed, 325 insertions(+) create mode 100644 contracts/go/clients/meshwantsclient/assignment.go create mode 100644 contracts/go/clients/meshwantsclient/assignment_test.go create mode 100644 contracts/go/clients/meshwantsclient/go.sum diff --git a/contracts/go/clients/meshwantsclient/assignment.go b/contracts/go/clients/meshwantsclient/assignment.go new file mode 100644 index 00000000..19a716b1 --- /dev/null +++ b/contracts/go/clients/meshwantsclient/assignment.go @@ -0,0 +1,176 @@ +package meshwantsclient + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "sync" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// RevisionVerb is the seat verb a part takes its first view of the assignment half by, as its manifest's invokes names +// it. +const RevisionVerb = "seat:" + seat.Seat + "." + seat.VerbRevision + +// kindTrunk is the kind of an assignment graph revision a running part acts on; a candidate's moves nothing. +const kindTrunk = "trunk" + +// Call asks one seat verb over the bus and answers what the holder said, whatever wraps it (process.Conn.Call, or +// stdio.Ask through the tool runner). +type Call func(key string, body any) (json.RawMessage, error) + +// assignmentCreated stands for the view's Created in its follower: a revision view is never made again, so one fixed +// time, never zero, lets the follower tell a revision heard from one it holds. +var assignmentCreated = time.Unix(0, 0).UTC() + +// assignmentViewName is how an assignment half's revision is named in a part's first-view status (views.Status.View, +// WaitsFor). +func assignmentViewName(revision int64) string { return strconv.FormatInt(revision, 10) } + +// Assignment is a part's view of the assignment half (design 47 G §245): a first view of the newest trunk revision asked +// of the seat mesh-wants' verb `revision`, then kept from its event `assignment-revision`, built on go/views' Follower, +// so the runtime's tools `_first_views` and `_first_view_again` read and steer it as every first view. +// Its picture is keyed `.`, each value that entry's reference. With no view the part acts on nothing: +// Revision says false, and a revision heard and not held yet is waited for, said in the status, and past +// views.WaitsBound raised as view-waits. +type Assignment struct { + Follower *views.Follower + // Held, when set, is told of each revision once the view holds it, on the ask's goroutine and outside any lock. + Held func(seat.AssignmentRevision) + + call Call + mu sync.Mutex + held *seat.AssignmentRevision + waits int64 +} + +// NewAssignment is an Assignment asking the seat through call. Nothing is asked before Start. +func NewAssignment(call Call) *Assignment { + a := &Assignment{call: call} + a.Follower = &views.Follower{Owner: seat.Seat, Tool: RevisionVerb, State: seat.StateAssignmentRevisions, Ask: a.ask} + return a +} + +// WireAssignment is an Assignment adopted by a stdio part's views.Reader: the reader writes its conditions and its +// view-waits finding, reads it as `mesh-wants.assignment-revisions`, lists and asks it again in its first-view tools, and +// at its Start subscribes the seat's event assignment-revision (the part's manifest consumes it) into Hear and asks the +// first view. held, when not nil, is the Assignment's Held. The part declares the seat's verb revision in invokes. +func WireAssignment(r *views.Reader, call Call, held func(seat.AssignmentRevision)) (*Assignment, error) { + a := NewAssignment(call) + a.Held = held + if err := r.Adopt(a.Follower, seat.Seat, seat.EventAssignmentRevision, a.Hear); err != nil { + return nil, err + } + return a, nil +} + +// WireAssignmentProcess is WireAssignment for a process's views.ProcessReader: its picture +// (`mesh-wants_assignment-revisions`) reads the half as a read-only bucket, and the seat's event assignment-revision on +// the module's durable consumer goes to Hear. +func WireAssignmentProcess(r *views.ProcessReader, call Call, held func(seat.AssignmentRevision)) (*Assignment, *views.Picture, error) { + a := NewAssignment(call) + a.Held = held + p, err := r.Adopt(a.Follower, seat.Seat, seat.EventAssignmentRevision, a.Hear) + if err != nil { + return nil, nil, err + } + return a, p, nil +} + +// Start asks the first view, the newest trunk revision, until it is answered or ctx ends. +func (a *Assignment) Start(ctx context.Context) error { return a.Follower.Start(ctx) } + +// Revision is the assignment graph revision the view holds; false while it has none, when the part acts on nothing new. +func (a *Assignment) Revision() (seat.AssignmentRevision, bool) { + a.mu.Lock() + defer a.mu.Unlock() + if a.held == nil { + return seat.AssignmentRevision{}, false + } + return *a.held, true +} + +// Hear takes one `assignment-revision` event, as the part's durable consumer of the seat's events hands it. A +// candidate's revision moves nothing; a trunk revision the view holds, or one older, is passed over; a newer one is +// waited for and asked on its own goroutine, so the consumer is never held while the holder is away. Data that is no +// such event is an error. +func (a *Assignment) Hear(ctx context.Context, data []byte) error { + var ev seat.AssignmentRevisionEvent + if err := json.Unmarshal(data, &ev); err != nil || ev.Revision <= 0 { + return fmt.Errorf("meshwantsclient: not a %s event: %s", seat.EventAssignmentRevision, clipBytes(data)) + } + if ev.Kind != kindTrunk { + return nil + } + a.mu.Lock() + held := int64(0) + if a.held != nil { + held = a.held.Revision + } + if ev.Revision > a.waits { + a.waits = ev.Revision + } + a.mu.Unlock() + if ev.Revision <= held { + return nil + } + // The follower keeps the newest revision heard, so an ask that was running when it was heard is asked once more. + a.Follower.Apply(views.Event{Owner: seat.Seat, State: seat.StateAssignmentRevisions, Revision: uint64(ev.Revision), Created: assignmentCreated, Gap: true}) + a.Follower.WaitFor(ctx, assignmentViewName(ev.Revision)) + go func() { + _ = a.Follower.Gap(ctx, fmt.Sprintf("assignment revision %d said, the view holds %d", ev.Revision, held)) + }() + return nil +} + +// ask is the Follower's Ask: the newest trunk revision, whole in one answer (its entries are named by digest). +func (a *Assignment) ask(_ context.Context, _ string) (views.Answer, error) { + raw, err := a.call(RevisionVerb, seat.RevisionInput{}) + if err != nil { + return views.Answer{}, err + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return views.Answer{}, fmt.Errorf("%s answered: %s", RevisionVerb, said) + } + var ans seat.RevisionAnswer + if err := json.Unmarshal(answer, &ans); err != nil { + return views.Answer{}, fmt.Errorf("%s answered what is not its answer: %s", RevisionVerb, clipBytes(answer)) + } + r := ans.Revision + if r.Revision <= 0 || r.Kind != kindTrunk { + return views.Answer{}, fmt.Errorf("%s answered a %q revision %d for the trunk", RevisionVerb, r.Kind, r.Revision) + } + values := make(map[string]json.RawMessage, len(r.Entries)) + for _, e := range r.Entries { + v, _ := json.Marshal(e) + values[e.Node+"."+e.Module] = v + } + a.mu.Lock() + moved := a.held == nil || a.held.Revision != r.Revision + a.held = &r + holds := a.waits <= r.Revision + if holds { + a.waits = 0 + } + a.mu.Unlock() + if holds { + a.Follower.Holds(assignmentViewName(r.Revision)) + } + if moved && a.Held != nil { + a.Held(r) + } + return views.Answer{V: 1, Owner: seat.Seat, State: seat.StateAssignmentRevisions, Revision: uint64(r.Revision), Created: assignmentCreated, Values: values, At: time.Now()}, nil +} + +func clipBytes(b []byte) string { + if len(b) > 200 { + return string(b[:200]) + "…" + } + return string(b) +} diff --git a/contracts/go/clients/meshwantsclient/assignment_test.go b/contracts/go/clients/meshwantsclient/assignment_test.go new file mode 100644 index 00000000..9d511a02 --- /dev/null +++ b/contracts/go/clients/meshwantsclient/assignment_test.go @@ -0,0 +1,116 @@ +package meshwantsclient + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "testing" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" +) + +// fakeWants answers the verb revision with its newest trunk revision, and counts each ask. +type fakeWants struct { + mu sync.Mutex + newest int64 + asks int +} + +func (f *fakeWants) call(key string, body any) (json.RawMessage, error) { + if key != RevisionVerb { + return nil, fmt.Errorf("asked %s", key) + } + if in := body.(seat.RevisionInput); in != (seat.RevisionInput{}) { + return nil, fmt.Errorf("asked %+v", in) + } + f.mu.Lock() + defer f.mu.Unlock() + f.asks++ + r := seat.AssignmentRevision{Revision: f.newest, Kind: kindTrunk, Placements: "p", + Catalogs: []seat.CatalogOf{{Branch: "main", Revision: 10 + f.newest}}, + Entries: []seat.NodeEntryRef{{Node: "anchor", Module: "nats", Digest: fmt.Sprintf("d%d", f.newest)}}} + return json.Marshal(seat.RevisionAnswer{Revision: r}) +} + +func (f *fakeWants) set(newest int64) { + f.mu.Lock() + defer f.mu.Unlock() + f.newest = newest +} + +func (f *fakeWants) asked() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.asks +} + +func revisionEvent(rev int64, kind string) []byte { + b, _ := json.Marshal(seat.AssignmentRevisionEvent{V: 1, Revision: rev, Kind: kind}) + return b +} + +// CT10 (design 47 G §245): a part takes a first view of the assignment half by the verb revision, then follows its +// event assignment-revision: a newer trunk revision is waited for, said in the status, and held; a candidate's, or one +// held, asks nothing; what is no such event is refused. +func TestTheAssignmentHalfIsAFirstViewThenItsEvents(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + held := make(chan seat.AssignmentRevision, 8) + next := func() seat.AssignmentRevision { + t.Helper() + select { + case r := <-held: + return r + case <-ctx.Done(): + t.Fatal("no revision held") + return seat.AssignmentRevision{} + } + } + f := &fakeWants{newest: 3} + a := NewAssignment(f.call) + a.Follower.Wait = time.Nanosecond + a.Held = func(r seat.AssignmentRevision) { held <- r } + if _, ok := a.Revision(); ok { + t.Fatal("a revision before its first view") + } + if err := a.Start(ctx); err != nil { + t.Fatal(err) + } + if r := next(); r.Revision != 3 || len(r.Catalogs) != 1 || r.Catalogs[0].Revision != 13 { + t.Fatalf("first view holds %+v", r) + } + if v, ok := a.Follower.Get("anchor.nats"); !ok || string(v) == "" { + t.Errorf("the picture lacks anchor.nats: %s", v) + } + + for _, ev := range [][]byte{revisionEvent(3, kindTrunk), revisionEvent(2, kindTrunk), revisionEvent(9, "candidate")} { + if err := a.Hear(ctx, ev); err != nil { + t.Fatal(err) + } + } + if n := f.asked(); n != 1 { + t.Errorf("a revision held, an older one or a candidate's asked again: %d asks", n) + } + + f.set(4) + if err := a.Hear(ctx, revisionEvent(4, kindTrunk)); err != nil { + t.Fatal(err) + } + if r := next(); r.Revision != 4 { + t.Fatalf("after assignment-revision 4 the view holds %d", r.Revision) + } + if s := a.Follower.Status(); s.View != "4" || s.WaitsFor != "" { + t.Errorf("status view %q, waits for %q", s.View, s.WaitsFor) + } + if r, ok := a.Revision(); !ok || r.Revision != 4 { + t.Errorf("Revision = %d, %v", r.Revision, ok) + } + + for _, data := range []string{`{}`, `not json`, `{"kind":"trunk"}`} { + if err := a.Hear(ctx, []byte(data)); err == nil { + t.Errorf("Hear(%s) took it", data) + } + } +} diff --git a/contracts/go/clients/meshwantsclient/go.mod b/contracts/go/clients/meshwantsclient/go.mod index 470f8994..88f343dc 100644 --- a/contracts/go/clients/meshwantsclient/go.mod +++ b/contracts/go/clients/meshwantsclient/go.mod @@ -6,3 +6,12 @@ require ( git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 ) + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) diff --git a/contracts/go/clients/meshwantsclient/go.sum b/contracts/go/clients/meshwantsclient/go.sum new file mode 100644 index 00000000..4ac8a82c --- /dev/null +++ b/contracts/go/clients/meshwantsclient/go.sum @@ -0,0 +1,24 @@ +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= From 9b9e9f68bb538c4938adb036c50981d5a696d927 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:48:34 +0100 Subject: [PATCH 07/16] TL4: the teller's manifest says version 0.1, declares its finding stream and hears the keeper's ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §383, §387, §368, §411 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-teller/manifest_test.go | 22 ++++++++++++++++--- modules/mesh-delivery-teller/module.json | 11 +++++++++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-teller/manifest_test.go b/modules/mesh-delivery-teller/manifest_test.go index 829136df..953960be 100644 --- a/modules/mesh-delivery-teller/manifest_test.go +++ b/modules/mesh-delivery-teller/manifest_test.go @@ -23,6 +23,13 @@ type state struct { History int `json:"history,omitempty"` } +type stream struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` +} + type artifact struct { Name string `json:"name"` Kind string `json:"kind"` @@ -44,6 +51,7 @@ type manifest struct { Invokes []string `json:"invokes"` Consumes []string `json:"consumes"` Emits []string `json:"emits"` + Streams []stream `json:"streams"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -102,7 +110,8 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1 if want := []string{"mesh-delivery-assigner.assigner_read_asks", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } - if want := []string{"mesh-delivery-assigner.views.asks"}; !reflect.DeepEqual(m.Consumes, want) { + // design 47 G §387: the keeper's ask for the findings is heard, and the finding stream is declared (§383). + if want := []string{"mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if want := []string{"operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) { @@ -122,8 +131,15 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1 if want := []string{tellerevents.AskEnded}; !reflect.DeepEqual(m.Emits, want) { t.Errorf("emits = %v, want %v", m.Emits, want) } - // No streams, bus or seat (ADR 0437 §4). - for _, k := range []string{"streams", "logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus"} { + // design 47 G §383, §387: the stream -findings, 7 days and 16 MiB; §368: version is MAJOR.MINOR. + if want := []stream{{Name: "mesh-delivery-teller-findings", Events: []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, MaxAgeDays: 7, MaxMiB: 16}}; !reflect.DeepEqual(m.Streams, want) { + t.Errorf("streams = %+v, want %+v", m.Streams, want) + } + if m.Version != "0.1" { + t.Errorf("version = %q, want 0.1", m.Version) + } + // No bus or seat (ADR 0437 §4). + for _, k := range []string{"logs", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the teller has none", k) } diff --git a/modules/mesh-delivery-teller/module.json b/modules/mesh-delivery-teller/module.json index c6e1a80c..cc176fd7 100644 --- a/modules/mesh-delivery-teller/module.json +++ b/modules/mesh-delivery-teller/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-teller", - "version": "1", + "version": "0.1", "slug": "teller", "tools": [ "teller_end_ask", @@ -12,8 +12,17 @@ "seat:mesh-hand-acts.hand-act" ], "consumes": [ + "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks" ], + "streams": [ + { + "name": "mesh-delivery-teller-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "emits": [ "ask-ended" ], From 3a4768ab43768b6460395a134ccaad93a9f7ae91 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:48:36 +0100 Subject: [PATCH 08/16] FB6a: the forge bridge reads trees, files, pulls and tags through the seat git and hears its pull and delete events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Taken over from the join's 473f9323, 5f40d230, 159a679b; the events are the seat's pull-updated, pull-merged, pull-closed and forge-delete. Builds: design 47 G §252, §310, §312, §317 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-delivery-forge-bridge-tools/main.go | 29 +-- .../main_test.go | 62 +++-- .../cmd/mesh-delivery-forge-bridge/call.go | 30 ++- .../cmd/mesh-delivery-forge-bridge/main.go | 4 +- .../mesh-delivery-forge-bridge/main_test.go | 120 +++++----- .../cmd/mesh-delivery-forge-bridge/open.go | 9 +- .../mesh-delivery-forge-bridge/open_test.go | 17 +- modules/mesh-delivery-forge-bridge/go.mod | 1 + .../internal/bridge/bridge.go | 162 +++++++------ .../internal/bridge/bridge_test.go | 214 ++++++++++-------- .../internal/bridge/call.go | 105 +++++---- .../internal/bridge/forgemade.go | 67 ++---- .../internal/bridge/tags.go | 66 ++---- .../internal/gitfixture/gitfixture.go | 42 ++++ .../manifest_test.go | 9 +- .../mesh-delivery-forge-bridge/module.json | 17 +- 16 files changed, 537 insertions(+), 417 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go index 0309cf12..d123a475 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -1,6 +1,6 @@ // Command mesh-delivery-forge-bridge-tools serves the forge bridge's tools (novox/hq ADR 0633 §4, issue 1585; design 51): -// the runtime launches it for a call. bridge_tags reads a package's git tags from the forge through the forge module's -// gitea_api, for the builder's floor. bridge_groups reads the delivery groups the gate holds and what the bridge kept of their ends; +// the runtime launches it for a call. bridge_tags reads a package's git tags through the seat git's verb tags, for the +// builder's floor. bridge_groups reads the delivery groups the gate holds and what the bridge kept of their ends; // bridge_group_end_again writes an ask into the bridge's state end-asks and waits for the bridge's answer there, since the // bridge is the other program and serves no request subject (ADR 0437 §4). package main @@ -43,18 +43,18 @@ const ( type asker func(key string, value []byte) (groups.EndAsk, error) // Tools are the bridge's tools: groupsRead reads the groups the gate holds with what the bridge kept of them. -func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string, number int) (json.RawMessage, bool, error), api bridge.API) []stdio.Tool { +func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string, number int) (json.RawMessage, bool, error), forge bridge.Caller) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: bridge.TagsTool, Description: "A package's git tags on the forge, for the builder's floor (design 51): every tag of repository whose name starts " + - "with prefix (go/v for a Go package in the go directory, v for one at the root), sorted. Read-only; read from the forge " + - "through the forge module's gitea_api, at most " + fmt.Sprint(bridge.TagsMaxPages*bridge.TagsPageSize) + " tags (more says so).", + "with prefix (go/v for a Go package in the go directory, v for one at the root), sorted. Read-only; read through the seat " + + "git's verb tags, at most " + fmt.Sprint(bridge.TagsMaxPages) + " pages (more says so).", Input: map[string]any{"type": "object", "required": []string{"repository"}, "properties": map[string]any{ "repository": map[string]any{"type": "string", "description": "/"}, "prefix": map[string]any{"type": "string", "description": "only tags starting with it"}}}, Run: func(args map[string]any) (any, error) { - return bridge.Tags(api, str(args, "repository"), str(args, "prefix")) + return bridge.Tags(context.Background(), forge, str(args, "repository"), str(args, "prefix")) }}, {Name: MergedTool, Description: "The merged message of one merge, asked again (issue 1585): the message the bridge wrote on its stream FORGE when the " + @@ -135,10 +135,10 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s // (bridge.MergedMessage), and, for a member of a delivery group, the group and its members as the process sets them. // // When the gate holds no merge with that commit, the merge was made on the forge (a recovery merge, the operator's own): -// the pull request is read from the forge through api (by number when given, else by the merge commit), the message is +// the pull request is read through the seat git's verb pull (by number when given, else by the merge commit), the message is // built the same way, and madeOnForge is true, for the answer and its log line to say. It is a lone pull request: no // group is set. -func MergedOf(raws map[string][]byte, repository, merge string, number int, trees bridge.Trees, files bridge.Files, api bridge.API) (out json.RawMessage, madeOnForge bool, err error) { +func MergedOf(raws map[string][]byte, repository, merge string, number int, trees bridge.Trees, files bridge.Files, forge bridge.Caller) (out json.RawMessage, madeOnForge bool, err error) { owner, repo, ok := strings.Cut(repository, "/") if !ok || owner == "" || repo == "" { return nil, false, fmt.Errorf("%q is not owner/name", repository) @@ -169,10 +169,10 @@ func MergedOf(raws map[string][]byte, repository, merge string, number int, tree out, err = forgemsg.EncodeForge(f) return out, false, err } - if api == nil { + if forge == nil { return nil, false, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, so its pull request is not known", repository, merge) } - fp, err := bridge.ForgeMade(api, owner, repo, number, merge) + fp, err := bridge.ForgeMade(context.Background(), forge, owner, repo, number, merge) if err != nil { return nil, false, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, and the forge does not give it as a merged pull request either: %w", repository, merge, err) } @@ -282,11 +282,6 @@ func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) return stdio.Ask(key, body) } -// forgeAPI is one GET of the forge's API through the forge module's gitea_api, the path the bridge reads the forge by. -func forgeAPI(path string) (json.RawMessage, error) { - return stdio.Ask("gitea.gitea_api", map[string]any{"path": path, "method": "GET"}) -} - func main() { merges, performed, asks := stdio.State(gateMergesState), stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState) tools := Tools(askEnd(asks, 25*time.Second), time.Now, @@ -303,8 +298,8 @@ func main() { } } ctx := context.Background() - return MergedOf(raws, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), forgeAPI) - }, forgeAPI) + return MergedOf(raws, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), askCaller) + }, askCaller) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go index 946eece1..616d23ea 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "encoding/json" "errors" "strings" @@ -9,11 +10,13 @@ import ( "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" + "mesh-delivery-forge-bridge/internal/gitfixture" "mesh-delivery-forge-bridge/internal/groups" ) @@ -136,7 +139,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { return nil, bridge.ErrNotFound } published := func(repo string, number int, head, branch, merge string) forgemsg.Forge { - body, _ := json.Marshal(map[string]any{"owner": "novox", "repo": repo, "number": number, "head": branch, "head_sha": head, "base": "main", "merge_commit_sha": merge}) + body, _ := json.Marshal(git.PullMergedEvent{Repository: "novox/" + repo, Number: int64(number), Branch: branch, Head: head, Base: "main", MergeCommit: merge}) msg, err := bridge.Convert(bridge.MergedSubject, body, trees, files) if err != nil { t.Fatal(err) @@ -158,7 +161,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { repository string merge string number int - api bridge.API + api bridge.Caller onForge bool want forgemsg.Forge fail string @@ -173,23 +176,25 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { // Issue 1966: a merge made on the forge (a recovery merge) has no merge in the gate; the bridge reads the pull request // from the forge, by number or by the merge commit, and checks its merge commit. {name: "a merge made on the forge, by number", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c2, number: 9, onForge: true, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", true, "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, + api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, {name: "a merge made on the forge, by the merge commit", raws: map[string][]byte{}, repository: "novox/a", merge: c2, onForge: true, - api: forgeAnswers(t, "/repos/novox/a/commits/"+c2+"/pull", true, "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, + api: forgeAnswers(t, 0, c2, "merged", "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, {name: "a forge pull request merged at another commit", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", true, "fix/x", h2, c1, "main"), fail: "not at " + c2}, + api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c1, "main"), fail: "not at " + c2}, {name: "a forge pull request not merged", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", false, "fix/x", h2, "", "main"), fail: "not merged"}, + api: forgeAnswers(t, 9, "", "open", "fix/x", h2, "", "main"), fail: "not merged"}, {name: "a forge pull request merged into another branch", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", true, "fix/x", h2, c2, "dev"), fail: "not the trunk"}, + api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "dev"), fail: "not the trunk"}, {name: "a forge pull request with no head branch", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: forgeAnswers(t, "/repos/novox/a/pulls/9", true, "refs/pull/9/head", h2, c2, "main"), fail: "no head branch"}, - {name: "a forge answer that is the bare body, no wrapper", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: func(string) (json.RawMessage, error) { return json.RawMessage(`{"number":9,"merged":true}`), nil }, fail: `{"result"`}, - {name: "a forge answer with no pull request number", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, - api: func(string) (json.RawMessage, error) { return json.RawMessage(`{"result":{"merged":true}}`), nil }, fail: "no number"}, + api: forgeAnswers(t, 9, "", "merged", "", h2, c2, "main"), fail: "no head branch"}, + {name: "a seat answer that says an error", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + api: answering(gitfixture.Given(t, "pull answers an error, never none", "verb:pull.answer")), fail: "holds no pull request"}, + {name: "a seat answer with no pull request", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + api: answering(json.RawMessage(`{}`)), fail: "carries nothing"}, {name: "a forge that does not answer", raws: map[string][]byte{}, repository: "novox/a", merge: c2, - api: func(string) (json.RawMessage, error) { return nil, errors.New("the forge is down") }, fail: "the forge is down"}, + api: func(context.Context, string, any) (json.RawMessage, error) { + return nil, errors.New("the forge is down") + }, fail: "the forge is down"}, } { t.Run(c.name, func(t *testing.T) { raw, onForge, err := MergedOf(c.raws, c.repository, c.merge, c.number, trees, files, c.api) @@ -216,21 +221,28 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { }) } -// forgeAnswers is the forge's gitea_api for one GET path: that pull request as the forge says it; any other path is not found. -func forgeAnswers(t *testing.T, path string, merged bool, branch, head, merge, base string) bridge.API { +// forgeAnswers is the seat git's verb pull for one pull request, asked by number (when number is given) or by the merge +// commit asked: the seat's own fixture with that pull request's state, branch, head, merge commit and base; any other +// ask is refused as the seat refuses it. +func forgeAnswers(t *testing.T, number int, byMerge, state, branch, head, merge, base string) bridge.Caller { t.Helper() - body, _ := json.Marshal(map[string]any{"number": 9, "merged": merged, "merge_commit_sha": merge, - "head": map[string]any{"ref": "refs/pull/9/head", "label": branch, "sha": head}, "base": map[string]any{"ref": base}}) - return func(p string) (json.RawMessage, error) { - if p != path { - return nil, errors.New("the forge has no " + p) + var a git.PullAnswer + if err := json.Unmarshal(gitfixture.Given(t, "pull answers", "verb:pull.answer"), &a); err != nil || a.Pull == nil { + t.Fatalf("the seat's pull fixture: %v", err) + } + p := *a.Pull + p.Repository, p.Number, p.State, p.Branch, p.Head, p.MergeCommit, p.Base = "novox/a", 9, state, branch, head, merge, base + answer, _ := json.Marshal(git.PullAnswer{Pull: &p}) + return func(_ context.Context, key string, body any) (json.RawMessage, error) { + in, ok := body.(git.PullInput) + if key != "seat:git.pull" || !ok || in.Repository != "novox/a" || int(in.Number) != number || in.MergeCommit != byMerge { + return json.RawMessage(`{"error":"the forge holds no such pull request"}`), nil } - return wrapped(body), nil + return answer, nil } } -// wrapped is gitea_api's answer for a body: {"result": } (mesh-catalog modules/gitea/tools/index.ts). -func wrapped(body []byte) json.RawMessage { - b, _ := json.Marshal(map[string]json.RawMessage{"result": body}) - return b +// answering is a seat that answers raw to every ask. +func answering(raw json.RawMessage) bridge.Caller { + return func(context.Context, string, any) (json.RawMessage, error) { return raw, nil } } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go index abf7f86b..fae52344 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/call.go @@ -19,7 +19,7 @@ const callTimeout = 30 * time.Second // caller calls a module's tool by its manifest key (`.`) with a JSON body and returns the tool's result. type caller func(ctx context.Context, key string, body any) (json.RawMessage, error) -// callerOn calls tools as the bridge's own principal. It copies mesh-sdk go/process/call.go (Call, callSubjects, +// callerOn calls tools and seat verbs as the bridge's own principal. It copies mesh-sdk go/process/call.go (Call, callSubjects, // request; hand-written here until the SDK's helper is used, issue 1242): the request is the tool's arguments as // JSON, published on the caller-named subject `mesh.mod..call..`, or on the subject that // names no caller, `mesh.mod..tool.`, when nothing answers the first; the reply is @@ -49,22 +49,40 @@ func callerOn(nc *nats.Conn, user string) caller { return r.Result, nil } return func(ctx context.Context, key string, body any) (json.RawMessage, error) { - module, tool, ok := strings.Cut(key, ".") - if !ok || module == "" || tool == "" { - return nil, fmt.Errorf("%q is not .", key) + called, fallback, err := callSubjects(key, token) + if err != nil { + return nil, err } data, err := json.Marshal(body) if err != nil { return nil, err } - result, err := request(ctx, "mesh.mod."+module+".call."+tool+"."+token, data) + result, err := request(ctx, called, data) if errors.Is(err, nats.ErrNoResponders) { - result, err = request(ctx, "mesh.mod."+module+".tool."+tool, data) + result, err = request(ctx, fallback, data) } return result, err } } +// callSubjects is the caller-named subject a key is called on and its plain fallback: a seat's verb +// (`seat:.`) on mesh.seat..call.., else a module's tool (`.`) on +// mesh.mod..call.. (mesh-sdk go/process/call.go callSubjectsOn). +func callSubjects(key, token string) (called, fallback string, err error) { + if rest, isSeat := strings.CutPrefix(key, "seat:"); isSeat { + seat, verb, ok := strings.Cut(rest, ".") + if !ok || seat == "" || verb == "" { + return "", "", fmt.Errorf("%q is not seat:.", key) + } + return "mesh.seat." + seat + ".call." + verb + "." + token, "mesh.seat." + seat + ".tool." + verb, nil + } + module, tool, ok := strings.Cut(key, ".") + if !ok || module == "" || tool == "" { + return "", "", fmt.Errorf("%q is not .", key) + } + return "mesh.mod." + module + ".call." + tool + "." + token, "mesh.mod." + module + ".tool." + tool, nil +} + // treesVia asks the forge for a commit's trees through call. func treesVia(ctx context.Context, call caller) bridge.Trees { return bridge.TreesVia(ctx, bridge.Caller(call)) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index e412765d..19ad49af 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -21,6 +21,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" @@ -197,7 +198,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The module's one pull of its durable consumer on the shared events stream is its event hub; the bridge has the one // handler, for every event its consumes filter lets through. Essential: the forge's events are what the bridge carries. hub := process.NewEventHub(logger.Printf) - if err := hub.Handle("mesh.mod.*.event.>", func(m jetstream.Msg) { + // The forge's events are the seat git's (novox/hq design 47 *The seat `git`*), whichever module holds it. + if err := hub.Handle("mesh.seat."+git.Seat+".event.>", func(m jetstream.Msg) { hear(&turn, func() { // Bounded: the turn is held, and no sync or other event goes on without it. hctx, cancel := context.WithTimeout(ctx, syncDeadline) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go index ede623fd..865a7c88 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-sdk/go/process" "github.com/nats-io/nats.go" @@ -21,6 +22,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" + "mesh-delivery-forge-bridge/internal/gitfixture" ) const head = "0123456789abcdef0123456789abcdef01234567" @@ -53,18 +55,14 @@ func (p *fakePub) Publish(_ context.Context, subject string, payload []byte, _ . return &jetstream.PubAck{}, nil } -// body is gitea.pull.updated as mesh-catalog modules/gitea/trunk.ts:233-250 writes it (the fields the bridge reads). +// body is the seat git's pull.updated (contracts/seats/git), the head head_sha laid over it when given. func body(m map[string]any) []byte { - base := map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "head": "feat/x", "head_sha": head, - "module_dirs": []string{"modules/a"}, "module_dirs_said": true} - for k, v := range m { - if v == nil { - delete(base, k) - } else { - base[k] = v - } + e := map[string]any{"repository": "novox/mesh-delivery", "number": 12, "branch": "feat/x", "head": head, "base": "main", + "change": map[string]any{"paths": []string{"modules/a/x.go"}, "module-dirs": []string{"modules/a"}}} + if h, ok := m["head_sha"]; ok { + e["head"] = h } - b, _ := json.Marshal(base) + b, _ := json.Marshal(e) return b } @@ -80,7 +78,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit } }) var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{} + m, pub := &fakeMsg{subject: git.SubjectPullUpdated, data: body(nil)}, &fakePub{} handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull" { t.Fatalf("published on %v", pub.subjects) @@ -96,7 +94,7 @@ func TestReadableEventIsWrittenOnForgeAndAcked(t *testing.T) { // 1235/AC2 (unit func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(map[string]any{"head_sha": "short"})}, &fakePub{} + m, pub := &fakeMsg{subject: git.SubjectPullUpdated, data: body(map[string]any{"head_sha": "short"})}, &fakePub{} handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if len(pub.subjects) != 0 { t.Errorf("a message was written: %v", pub.subjects) @@ -104,13 +102,13 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 if m.termd != 1 || m.acked != 0 { t.Errorf("terminated %d acked %d, want terminated once", m.termd, m.acked) } - if !strings.Contains(logs.String(), "mesh.mod.gitea.event.pull.updated") || !strings.Contains(logs.String(), "head") { + if !strings.Contains(logs.String(), git.SubjectPullUpdated) || !strings.Contains(logs.String(), "head") { t.Errorf("log %q names neither the subject nor why", logs.String()) } // 1235: an unwritten message is not acked (a failed write is redelivered by the server, not retried here); merged here // from a test of its own, for the test budget. logs.Reset() - m, pub = &fakeMsg{subject: "mesh.mod.gitea.event.pull.updated", data: body(nil)}, &fakePub{err: errors.New("no stream")} + m, pub = &fakeMsg{subject: git.SubjectPullUpdated, data: body(nil)}, &fakePub{err: errors.New("no stream")} handle(context.Background(), m, pub, nil, someFiles, nil, log.New(&logs, "", 0)) if m.acked != 0 || m.termd != 0 { t.Errorf("acked %d terminated %d, want neither", m.acked, m.termd) @@ -120,16 +118,16 @@ func TestUnreadableEventIsTerminatedAndLogged(t *testing.T) { // 1235/AC5 } } -// mergedEvent is gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 writes it (the fields the bridge reads). +// mergedEvent is the seat git's pull.merged. func mergedEvent() []byte { - b, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "head": "feat/x", "head_sha": head, - "base": "main", "merge_commit_sha": "89abcdef0123456789abcdef0123456789abcdef"}) + b, _ := json.Marshal(map[string]any{"repository": "novox/mesh-delivery", "number": 12, "branch": "feat/x", "head": head, + "base": "main", "merge-commit": "89abcdef0123456789abcdef0123456789abcdef"}) return b } func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/AC3 var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{} + m, pub := &fakeMsg{subject: git.SubjectPullMerged, data: mergedEvent()}, &fakePub{} call := func(context.Context, string, any) (json.RawMessage, error) { return nil, errors.New("the tree listing of novox/mesh-delivery was cut") } @@ -147,33 +145,36 @@ func TestTreesRefusedLeavesTheEventForTenSecondsAndLogs(t *testing.T) { // 1236/ } } -func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/AC1 (through the tool's request and answer) +func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/AC1; design 47 *The seat git* (verbs trees, file) var logs bytes.Buffer - m, pub := &fakeMsg{subject: "mesh.mod.gitea.event.pull.merged", data: mergedEvent()}, &fakePub{} + m, pub := &fakeMsg{subject: git.SubjectPullMerged, data: mergedEvent()}, &fakePub{} var keys []string var args []any - var mu sync.Mutex // the files are asked for concurrently (issue 1458) + var mu sync.Mutex // the files are asked for concurrently (issue 1458) + trees := gitfixture.Given(t, "trees answers", "verb:trees.answer") // root ddd…, "" ddd…, contracts eee… + file := gitfixture.Given(t, "file answers", "verb:file.answer") call := func(_ context.Context, key string, body any) (json.RawMessage, error) { mu.Lock() keys, args = append(keys, key), append(args, body) mu.Unlock() - if key == "gitea.gitea_get_file" { // the manifests, 1448; the trees are what this test is about - return json.RawMessage(`{"file":{"path":"p","sha":"0123","size":2,"content":"{}"}}`), nil + if key == "seat:git.file" { // the manifests, 1448; the trees are what this test is about + return file, nil } - return json.RawMessage(`{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`), nil + return trees, nil } handle(context.Background(), m, pub, treesVia(context.Background(), call), filesVia(context.Background(), call), nil, log.New(&logs, "", 0)) - if len(keys) == 0 || keys[0] != "gitea.gitea_commit_trees" || slices.Index(keys[1:], "gitea.gitea_commit_trees") >= 0 { + if len(keys) == 0 || keys[0] != "seat:git.trees" || slices.Index(keys[1:], "seat:git.trees") >= 0 { t.Fatalf("calls %v, want the trees once and first", keys) } - if got, _ := json.Marshal(args[0]); string(got) != `{"owner":"novox","repo":"mesh-delivery","sha":"89abcdef0123456789abcdef0123456789abcdef"}` { + if got, _ := json.Marshal(args[0]); !sameJSON(got, []byte(`{"repository":"novox/mesh-delivery","commit":"89abcdef0123456789abcdef0123456789abcdef"}`)) { t.Errorf("arguments %s", got) } if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged" { t.Fatalf("published on %v", pub.subjects) } f, err := forgemsg.DecodeForge(pub.data[0]) - if err != nil || f.Root != "r0" || f.Trees[""] != "t0" || f.Trees["modules/a"] != "t1" || f.Merge == "" { + d, e := strings.Repeat("d", 40), strings.Repeat("e", 40) + if err != nil || f.Root != d || f.Trees[""] != d || f.Trees["contracts"] != e || f.Merge == "" { t.Errorf("message %+v err %v", f, err) } if m.acked != 1 { @@ -181,60 +182,65 @@ func TestMergeCallsTheToolOnceAndWritesTheMergedMessage(t *testing.T) { // 1236/ } } -func TestAnAnswerWithoutARootIsNotUsed(t *testing.T) { // 1236 (a shape the tool does not answer) - call := func(context.Context, string, any) (json.RawMessage, error) { - return json.RawMessage(`{"trees":{"a":"t"}}`), nil - } - if _, _, err := treesVia(context.Background(), call)("o", "r", "s"); err == nil { - t.Error("an answer with no root was taken") - } - call = func(context.Context, string, any) (json.RawMessage, error) { return json.RawMessage(`[1]`), nil } - if _, _, err := treesVia(context.Background(), call)("o", "r", "s"); err == nil { - t.Error("an answer that is not an object was taken") +func TestAnAnswerWithoutARootIsNotUsed(t *testing.T) { // 1236 (a shape the seat does not answer); design 47 *A failed read answers an error* + for _, raw := range []json.RawMessage{json.RawMessage(`{"trees":{"a":"t"}}`), json.RawMessage(`[1]`), + gitfixture.Given(t, "trees answers an error, never none", "verb:trees.answer")} { + call := func(context.Context, string, any) (json.RawMessage, error) { return raw, nil } + if _, _, err := treesVia(context.Background(), call)("o", "r", "s"); err == nil { + t.Errorf("the answer %s was taken", raw) + } } } -// The answer of gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile write it, and the -// error of a path the forge does not have as client.ts request writes it (`Gitea API ${path}: ${status} ${text}`). -const getFileAnswer = `{"file":{"path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","sha":"0123","size":4,"content":"x y\n"}}` -const getFile404 = "Gitea API /repos/novox/mesh-delivery/contents/modules/a/go.sum?ref=89abcdef: 404 {\"message\":\"GetContentsOrList\",\"url\":\"https://forge.example/api/swagger\"}" - -func TestFilesViaAsksByOwnerRepoPathRefAndTakesTheContent(t *testing.T) { // 1448/AC4 +func TestFilesViaAsksByOwnerRepoPathRefAndTakesTheContent(t *testing.T) { // 1448/AC4; design 47 *The seat git* (verb file) var keys []string var args []any call := func(_ context.Context, key string, body any) (json.RawMessage, error) { keys, args = append(keys, key), append(args, body) - return json.RawMessage(getFileAnswer), nil + return gitfixture.Given(t, "file answers", "verb:file.answer"), nil } - got, err := filesVia(context.Background(), call)("novox", "mesh-delivery", "modules/a/go.sum", "89abcdef0123456789abcdef0123456789abcdef") - if err != nil || string(got) != "x y\n" { + got, err := filesVia(context.Background(), call)("novox", "mesh-contracts", "seats/index.json", strings.Repeat("a", 40)) + if err != nil || string(got) != `{"format":1,"seats":["mesh-broker"]}` { t.Fatalf("got %q err %v", got, err) } - if len(keys) != 1 || keys[0] != "gitea.gitea_get_file" { + if len(keys) != 1 || keys[0] != "seat:git.file" { t.Fatalf("calls %v", keys) } - if b, _ := json.Marshal(args[0]); string(b) != `{"owner":"novox","path":"modules/a/go.sum","ref":"89abcdef0123456789abcdef0123456789abcdef","repo":"mesh-delivery"}` { - t.Errorf("arguments %s", b) + want := gitfixture.Given(t, "file asks", "verb:file.input") + if b, _ := json.Marshal(args[0]); !sameJSON(b, want) { + t.Errorf("arguments %s, want the seat's %s", b, want) } } -func TestFilesViaTellsNotFoundFromOtherErrorsAndOtherShapes(t *testing.T) { // 1448/AC3 - call := func(err error, raw string) caller { - return func(context.Context, string, any) (json.RawMessage, error) { return json.RawMessage(raw), err } +func TestFilesViaTellsNotFoundFromOtherErrorsAndOtherShapes(t *testing.T) { // 1448/AC3; design 47 *A failed read answers an error* + call := func(err error, raw json.RawMessage) caller { + return func(context.Context, string, any) (json.RawMessage, error) { return raw, err } } - if _, err := filesVia(context.Background(), call(errors.New(getFile404), ""))("o", "r", "p", "s"); !errors.Is(err, bridge.ErrNotFound) { - t.Errorf("a 404 is %v, want ErrNotFound", err) + absent := gitfixture.All(t, "file answers", "verb:file.answer")[1] + if _, err := filesVia(context.Background(), call(nil, absent))("o", "r", "p", "s"); !errors.Is(err, bridge.ErrNotFound) { + t.Errorf("an absent file is %v, want ErrNotFound", err) } - if _, err := filesVia(context.Background(), call(errors.New("Gitea API /repos/o/r/contents/p: 502 bad gateway"), ""))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) { - t.Errorf("a 502 is %v, want an error that is not ErrNotFound", err) + if _, err := filesVia(context.Background(), call(errors.New("nats: timeout"), nil))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) { + t.Errorf("a seat that does not answer is %v, want an error that is not ErrNotFound", err) } - for _, raw := range []string{`{"file":null}`, `{}`, `[1]`, `{"file":{"path":"p"}}`} { + for _, raw := range []json.RawMessage{gitfixture.Given(t, "file answers an error, never none", "verb:file.answer"), json.RawMessage(`{}`), json.RawMessage(`[1]`), json.RawMessage(`{"path":"p"}`)} { if _, err := filesVia(context.Background(), call(nil, raw))("o", "r", "p", "s"); err == nil || errors.Is(err, bridge.ErrNotFound) { t.Errorf("the answer %s gave %v, want an error", raw, err) } } } +// sameJSON is whether two JSON texts say the same value. +func sameJSON(a, b []byte) bool { + var x, y any + if json.Unmarshal(a, &x) != nil || json.Unmarshal(b, &y) != nil { + return false + } + ax, _ := json.Marshal(x) + by, _ := json.Marshal(y) + return bytes.Equal(ax, by) +} + // someFiles is a forge that has a module.json at every path and no other file, for the tests that are not about the files. func someFiles(_, _, path, _ string) ([]byte, error) { if strings.HasSuffix(path, "module.json") { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go index a93543dd..93f4d07b 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open.go @@ -10,14 +10,15 @@ import ( "github.com/nats-io/nats.go/jetstream" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" ) -// openPullsTool is the forge's tool for every open pull request (mesh-catalog modules/gitea/tools/index.ts -// gitea_open_pulls: no arguments, answers {pulls: [...]}). -const openPullsTool = "gitea.gitea_open_pulls" +// openPullsTool is the seat git's verb for every open pull request (novox/hq design 47 *The seat `git`*): no arguments, +// answers {pulls: [...]} or an error. +var openPullsTool = bridge.GitKey(git.VerbOpenPulls) // headStore is the bucket heads: the head last written on FORGE for each pull request, by bridge.Track's key // (novox/hq ADR 0443 §5). @@ -84,7 +85,7 @@ func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore, defer mu.Unlock() ctx, cancel := context.WithTimeout(ctx, syncDeadline) defer cancel() - raw, err := call(ctx, openPullsTool, struct{}{}) + raw, err := call(ctx, openPullsTool, git.OpenPullsInput{}) if err != nil { return fmt.Errorf("%s: %w", openPullsTool, err) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go index a9caade0..5861b9b7 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/open_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "log" + "strconv" "strings" "sync" "testing" @@ -16,12 +17,14 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) -// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds its entries inside {pulls}. +// openAnswer is the seat git's answer of open-pulls: two into the trunk, one into another branch. func openAnswer() json.RawMessage { - return json.RawMessage(`{"pulls":[ - {"owner":"novox","repo":"mesh-delivery","number":12,"base":"main","head":"feat/x","head_sha":"` + head + `","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}, - {"owner":"novox","repo":"mesh-catalog","number":300,"base":"main","head":"feat/x","head_sha":"` + strings.Repeat("a", 40) + `","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}, - {"owner":"novox","repo":"mesh-catalog","number":301,"base":"feat/y","head":"feat/z","head_sha":"` + strings.Repeat("b", 40) + `","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}]}`) + entry := func(repo string, number int, base, branch, sha string) string { + return `{"pull":{"repository":"novox/` + repo + `","number":` + strconv.Itoa(number) + `,"base":"` + base + `","branch":"` + branch + + `","head":"` + sha + `","state":"open","url":"h"},"required":[],"checks":{}}` + } + return json.RawMessage(`{"pulls":[` + entry("mesh-delivery", 12, "main", "feat/x", head) + `,` + + entry("mesh-catalog", 300, "main", "feat/x", strings.Repeat("a", 40)) + `,` + entry("mesh-catalog", 301, "feat/y", "feat/z", strings.Repeat("b", 40)) + `]}`) } func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1, AC4 @@ -35,7 +38,7 @@ func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1, if err := runSync(call, pub, newHeads(), &sync.Mutex{}, log.New(&logs, "", 0)); err != nil { t.Fatal(err) } - if len(keys) != 1 || keys[0] != "gitea.gitea_open_pulls" { + if len(keys) != 1 || keys[0] != "seat:git.open-pulls" { t.Errorf("calls %v", keys) } if len(pub.subjects) != 3 { @@ -232,7 +235,7 @@ func TestSyncOpenFailureNamesTheCallAndTheForgesWords(t *testing.T) { // 1237/AC return nil, errors.New("the forge said 502") } err := runSync(call, pub, newHeads(), &sync.Mutex{}, log.New(&bytes.Buffer{}, "", 0)) - if err == nil || !strings.Contains(err.Error(), "gitea.gitea_open_pulls") || !strings.Contains(err.Error(), "the forge said 502") { + if err == nil || !strings.Contains(err.Error(), "seat:git.open-pulls") || !strings.Contains(err.Error(), "the forge said 502") { t.Errorf("err = %v", err) } if len(pub.subjects) != 0 { diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 042b8ff1..9a5a262e 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -4,6 +4,7 @@ go 1.26.0 require ( git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go index a920e158..1a7a7439 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge.go @@ -1,4 +1,4 @@ -// Package bridge turns the forge module's pull request events into the messages the forge bridge writes on FORGE. +// Package bridge turns the seat git's pull request events into the messages the forge bridge writes on FORGE. package bridge import ( @@ -15,6 +15,7 @@ import ( "strings" "sync" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" @@ -35,13 +36,13 @@ var ErrRetry = errors.New("the forge could not say") const Trunk = "main" // Trees asks the forge for the trees of a commit: the root tree and the tree of every directory holding a module.json -// (the root directory spelled ""), as the forge's gitea_commit_trees answers (issue 1234/AC4). +// (the root directory spelled ""), as the seat git's verb trees answers (issue 1234/AC4). type Trees func(owner, repo, sha string) (root string, trees map[string]string, err error) // ErrNotFound says the forge has no file at that path and ref (the forge's "404", issue 1448). var ErrNotFound = errors.New("the forge has no such file") -// Files asks the forge for one file's bytes at a ref (a commit), as the forge's gitea_get_file answers; a file the forge +// Files asks the forge for one file's bytes at a ref (a commit), as the seat git's verb file answers; a file the forge // says is not there is ErrNotFound, any other failure any other error (issue 1448). type Files func(owner, repo, path, ref string) ([]byte, error) @@ -64,16 +65,16 @@ type Message struct { Said string } -// The subjects of the forge module's events the bridge writes pull messages for (mesh-catalog modules/gitea, -// published by the module as mesh.mod..event.). +// The subjects of the seat git's events the bridge writes pull messages for (novox/hq design 47 *The seat `git`*): the +// seat's holder says them on mesh.seat.git.event., whichever module holds it; the bridge names none. const ( - UpdatedSubject = "mesh.mod.gitea.event.pull.updated" - ClosedSubject = "mesh.mod.gitea.event.pull.closed" - MergedSubject = "mesh.mod.gitea.event.pull.merged" - // DeletedSubject is the forge module's forge.delete: a branch or a tag deleted (mesh-catalog modules/gitea - // webhook.ts: owner, repo, ref, ref_type, sha). Only the forge module's account may publish it, the subject being - // in its own namespace (design 32 § 2), so no other module can end a branch's tagged versions. - DeletedSubject = "mesh.mod.gitea.event.forge.delete" + UpdatedSubject = git.SubjectPullUpdated + ClosedSubject = git.SubjectPullClosed + MergedSubject = git.SubjectPullMerged + // DeletedSubject is the seat's forge.delete: a branch or a tag deleted (repository, ref, ref-type). Only the seat's + // holder may publish on the seat's subjects (novox/hq ADR 0363 §2), so no other module can end a branch's tagged + // versions. + DeletedSubject = git.SubjectForgeDelete ) // ForgeBranchDeleted is the kind of a branch-deleted message on FORGE (novox/hq design 51 *The forge bridge says a @@ -91,21 +92,14 @@ type BranchDeleted struct { Head string `json:"head,omitempty"` } -// deleted is the part of forge.delete the bridge reads. -type deleted struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - Ref string `json:"ref"` - RefType string `json:"ref_type"` - SHA string `json:"sha"` -} - // branchDeleted turns forge.delete into a branch-deleted message; a deleted tag is not handled. func branchDeleted(subject string, data []byte) (Message, error) { - var d deleted - if err := json.Unmarshal(data, &d); err != nil { + var e git.ForgeDeleteEvent + if err := json.Unmarshal(data, &e); err != nil { return Message{}, unreadable(subject, "the body is not json: %v", err) } + d := struct{ Owner, Repo, Ref, RefType string }{Ref: e.Ref, RefType: e.RefType} + d.Owner, d.Repo, _ = strings.Cut(e.Repository, "/") if d.RefType != "branch" { return Message{}, fmt.Errorf("%w: %s of a %s (%s)", ErrNotHandled, subject, d.RefType, d.Ref) } @@ -118,7 +112,7 @@ func branchDeleted(subject string, data []byte) (Message, error) { case branch == Trunk: return Message{}, fmt.Errorf("%w: %s of the trunk %s", ErrNotHandled, subject, Trunk) } - body, err := json.Marshal(BranchDeleted{V: keys.Version, Kind: ForgeBranchDeleted, Repository: d.Owner + "/" + d.Repo, Branch: branch, Head: d.SHA}) + body, err := json.Marshal(BranchDeleted{V: keys.Version, Kind: ForgeBranchDeleted, Repository: d.Owner + "/" + d.Repo, Branch: branch}) if err != nil { return Message{}, unreadable(subject, "%v", err) } @@ -128,22 +122,48 @@ func branchDeleted(subject string, data []byte) (Message, error) { var sha40 = regexp.MustCompile(`^[0-9a-f]{40}$`) -// pull is the part of gitea.pull.merged (index.ts:186-207), gitea.pull.updated (mesh-catalog modules/gitea/trunk.ts:233-250) and gitea.pull.closed -// (index.ts:127-128) the bridge reads; head is the branch and head_sha the commit. +// pull is the part of the seat git's pull.updated, pull.merged and pull.closed the bridge reads: Head is the branch and +// HeadSHA the commit, Owner and Repo the repository's two halves. type pull struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - Number int `json:"number"` - Head string `json:"head"` - HeadSHA string `json:"head_sha"` - ModuleDirs []string `json:"module_dirs"` - ModuleDirsSaid bool `json:"module_dirs_said"` - Base string `json:"base"` - MergeSHA string `json:"merge_commit_sha"` - // Paths are the files the pull request changes (a removed or renamed one included), PathsTruncated says the forge - // cut the list; nil when the forge did not say them. - Paths *[]string `json:"paths"` - PathsTruncated bool `json:"paths_truncated"` + Owner, Repo string + Number int + Head, HeadSHA string + ModuleDirs []string + ModuleDirsSaid bool + Base string + MergeSHA string +} + +// pullOf reads one of the seat's pull events. The directories are the seat's word only when its change says them: an +// absent `module-dirs` is "could not be read", never "none". +func pullOf(data []byte) (pull, error) { + var e struct { + Repository string `json:"repository"` + Number int64 `json:"number"` + Branch string `json:"branch"` + Head string `json:"head"` + Base string `json:"base"` + MergeCommit string `json:"merge-commit"` + Change json.RawMessage `json:"change"` + } + if err := json.Unmarshal(data, &e); err != nil { + return pull{}, err + } + p := pull{Number: int(e.Number), Head: e.Branch, HeadSHA: e.Head, Base: e.Base, MergeSHA: e.MergeCommit} + p.Owner, p.Repo, _ = strings.Cut(e.Repository, "/") + if len(e.Change) > 0 && string(e.Change) != "null" { + var c map[string]json.RawMessage + if err := json.Unmarshal(e.Change, &c); err != nil { + return pull{}, fmt.Errorf("its change: %w", err) + } + if raw, said := c["module-dirs"]; said { + if err := json.Unmarshal(raw, &p.ModuleDirs); err != nil { + return pull{}, fmt.Errorf("its change's module-dirs: %w", err) + } + p.ModuleDirsSaid = true + } + } + return p, nil } func unreadable(subject, why string, args ...any) error { @@ -176,12 +196,12 @@ func ConvertKept(subject string, data []byte, trees Trees, files Files, kept Kep default: return Message{}, fmt.Errorf("%w: %s", ErrNotHandled, subject) } - var p pull - if err := json.Unmarshal(data, &p); err != nil { - return Message{}, unreadable(subject, "the body is not json: %v", err) + p, err := pullOf(data) + if err != nil { + return Message{}, unreadable(subject, "the body is not the seat's json: %v", err) } switch { - case p.Owner == "" || p.Repo == "": + case p.Owner == "" || p.Repo == "" || strings.Contains(p.Repo, "/"): return Message{}, unreadable(subject, "no owner or repo") case p.Number <= 0: return Message{}, unreadable(subject, "no pull request number") @@ -462,29 +482,39 @@ func merged(subject string, p pull, trees Trees, files Files, kept Kept) (Messag return msg, nil } -// openPull is an entry of gitea_open_pulls's answer (mesh-catalog modules/gitea/client.ts:369-381 openPulls, with the -// branch as `head`, issue 1234/AC5), the part the bridge reads. -type openPull struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - Number int `json:"number"` - Base string `json:"base"` - Head string `json:"head"` - HeadSHA string `json:"head_sha"` +// openPulls reads the seat git's answer of open-pulls: an answer saying an error is the error, never none open. +func openPulls(answer []byte) ([]pull, error) { + var a git.OpenPullsAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return nil, fmt.Errorf("the seat git's answer of open-pulls is not its contract's: %v", err) + } + if a.Error != "" { + return nil, fmt.Errorf("the seat git refused open-pulls: %s", a.Error) + } + var said map[string]json.RawMessage + if _ = json.Unmarshal(answer, &said); said["pulls"] == nil || string(said["pulls"]) == "null" { + // None open is `pulls: []`; an answer naming no list would drop every candidate on a holder's fault. + return nil, fmt.Errorf("open-pulls: %w", errNoAnswer) + } + out := make([]pull, 0, len(a.Pulls)) + for _, o := range a.Pulls { + p := pull{Number: int(o.Pull.Number), Head: o.Pull.Branch, HeadSHA: o.Pull.Head, Base: o.Pull.Base} + p.Owner, p.Repo, _ = strings.Cut(o.Pull.Repository, "/") + out = append(out, p) + } + return out, nil } -// OpenPulls turns the answer of gitea_open_pulls, {pulls: [...]}, into one pull message per open pull request into +// OpenPulls turns the seat git's answer of open-pulls into one pull message per open pull request into // the trunk, action updated and its directories not known (the forge's answer does not say them). A pull request into // another branch gets none. An entry that cannot be read is named in skipped and gets none; an answer that is not -// {pulls} is an error. +// the contract's, or that says an error, is an error. func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) { - var a struct { - Pulls *[]openPull `json:"pulls"` + pulls, err := openPulls(answer) + if err != nil { + return nil, nil, err } - if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil { - return nil, nil, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err) - } - for _, p := range *a.Pulls { + for _, p := range pulls { if p.Base != Trunk { continue } @@ -523,18 +553,16 @@ func Track(f forgemsg.Forge) (key, head string, ok bool) { return "", "", false } -// OpenMessage turns the answer of gitea_open_pulls into the open message: every open pull request into the trunk, by +// OpenMessage turns the seat git's answer of open-pulls into the open message: every open pull request into the trunk, by // repository and number, whether its head reads or not and whether FORGE holds it or not, in repository and number // order. A pull request into another branch is not named. An answer that names none is a message too: nothing is open. func OpenMessage(answer []byte) (Message, error) { - var a struct { - Pulls *[]openPull `json:"pulls"` - } - if err := json.Unmarshal(answer, &a); err != nil || a.Pulls == nil { - return Message{}, fmt.Errorf("the answer of gitea_open_pulls is not {pulls: [...]}: %v", err) + pulls, err := openPulls(answer) + if err != nil { + return Message{}, err } open := []forgemsg.OpenPull{} - for _, p := range *a.Pulls { + for _, p := range pulls { if p.Base != Trunk || p.Owner == "" || p.Repo == "" || p.Number <= 0 { continue } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go index 6744636b..2ab29a65 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/bridge_test.go @@ -1,6 +1,7 @@ package bridge import ( + "context" "encoding/json" "errors" "strings" @@ -8,44 +9,63 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/keys" + + "mesh-delivery-forge-bridge/internal/gitfixture" ) const head = "0123456789abcdef0123456789abcdef01234567" -// updated is the body of gitea.pull.updated as mesh-catalog modules/gitea/trunk.ts:233-250 (announcementOf) writes -// it: owner, repo, number, base_sha, title, body, base, head (the branch), head_sha, clone_url, html_url, paths, -// paths_truncated, removed, module_dirs, module_dirs_said, merge_check, merge_check_said. +// updated is the body of the seat git's pull.updated (contracts/seats/git): the pull request, its base's head and its +// change. extra names the parts a test changes by their short names (owner, repo, number, head for the branch, +// head_sha for the commit, base, module_dirs, module_dirs_said), a nil value leaving the part out (seatPull). func updated(extra map[string]any) []byte { - m := map[string]any{ - "owner": "novox", "repo": "mesh-delivery", "number": 12, - "base_sha": "ffffffffffffffffffffffffffffffffffffffff", "title": "A change", "body": "text", "base": "main", - "head": "feat/x", "head_sha": head, "clone_url": "https://forge.example/novox/mesh-delivery.git", - "html_url": "https://forge.example/novox/mesh-delivery/pulls/12", - "paths": []string{"modules/a/x.go"}, "paths_truncated": false, "removed": []string{}, - "module_dirs": []string{"modules/a"}, "module_dirs_said": true, - "merge_check": true, "merge_check_said": true, - } - for k, v := range extra { - if v == nil { - delete(m, k) - } else { - m[k] = v + return seatPull(map[string]any{"base_sha": "ffffffffffffffffffffffffffffffffffffffff", + "paths": []string{"modules/a/x.go"}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true}, extra) +} + +// closed is the body of the seat git's pull.closed. +func closed() []byte { return seatPull(nil, nil) } + +// seatPull is a pull event of the seat git: the pull request novox/mesh-delivery#12 of feat/x at head into main, with +// the parts of base and extra laid over it by their short names, a nil value leaving the part out. module_dirs is said +// in the change only when module_dirs_said is true: its absence is "could not be read". +func seatPull(base, extra map[string]any) []byte { + m := map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "A change", "head": "feat/x", + "head_sha": head, "base": "main", "html_url": "https://forge.example/novox/mesh-delivery/pulls/12"} + for _, over := range []map[string]any{base, extra} { + for k, v := range over { + if v == nil { + delete(m, k) + } else { + m[k] = v + } } } - b, _ := json.Marshal(m) + e := map[string]any{} + owner, _ := m["owner"].(string) + repo, _ := m["repo"].(string) + if owner != "" || repo != "" { + e["repository"] = owner + "/" + repo + } + for short, name := range map[string]string{"number": "number", "title": "title", "head": "branch", "head_sha": "head", + "base": "base", "html_url": "url", "merge_commit_sha": "merge-commit", "base_sha": "base-head", "merged_at": "merged-at"} { + if v, ok := m[short]; ok { + e[name] = v + } + } + if paths, ok := m["paths"]; ok { + c := map[string]any{"paths": paths} + if m["module_dirs_said"] == true && m["module_dirs"] != nil { + c["module-dirs"] = m["module_dirs"] + } + e["change"] = c + } + b, _ := json.Marshal(e) return b } -// closed is the body of gitea.pull.closed as mesh-catalog modules/gitea/index.ts:127-128 writes it: owner, repo, -// number, title, head, head_sha, base, html_url. -func closed() []byte { - b, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "A change", - "head": "feat/x", "head_sha": head, "base": "main", "html_url": "https://forge.example/novox/mesh-delivery/pulls/12"}) - return b -} - -const updatedSubject = "mesh.mod.gitea.event.pull.updated" -const closedSubject = "mesh.mod.gitea.event.pull.closed" +const updatedSubject = "mesh.seat.git.event.pull-updated" +const closedSubject = "mesh.seat.git.event.pull-closed" func TestUpdatedBecomesAPullMessage(t *testing.T) { // 1235/AC2 (the conversion; the bus is the integration test) msg, err := Convert(updatedSubject, updated(nil), nil, anyFiles) @@ -142,8 +162,8 @@ func TestUnreadableEventsWriteNothing(t *testing.T) { // 1235/AC5 "upper-case head": {updatedSubject, updated(map[string]any{"head_sha": strings.ToUpper(head)}), "head"}, "no branch": {updatedSubject, updated(map[string]any{"head": nil}), "branch"}, "not json": {updatedSubject, []byte("{"), "json"}, - "closed no head": {closedSubject, mustDrop(closed(), "head_sha"), "head"}, - "closed no repo": {closedSubject, mustDrop(closed(), "repo"), "repo"}, + "closed no head": {closedSubject, mustDrop(closed(), "head"), "head"}, + "closed no repo": {closedSubject, mustDrop(closed(), "repository"), "repo"}, } { msg, err := Convert(c.subject, c.data, nil, anyFiles) if err == nil { @@ -171,21 +191,21 @@ func mustDrop(body []byte, key string) []byte { } func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236 - for _, s := range []string{"mesh.mod.gitea.event.something.else", "mesh.mod.other.event.pull.updated", "mesh.mod.other.event.forge.delete"} { + for _, s := range []string{"mesh.seat.git.event.something.else", "mesh.mod.gitea.event.pull.updated", "mesh.mod.other.event.forge.delete"} { if _, err := Convert(s, updated(nil), nil, anyFiles); !errors.Is(err, ErrNotHandled) { t.Errorf("%s: err = %v, want ErrNotHandled", s, err) } } - // The forge module's forge.delete of a branch is a branch-deleted message (DLV-FB1, design 51); of a tag or the + // The seat's forge.delete of a branch is a branch-deleted message (DLV-FB1, design 51); of a tag or the // trunk it is not handled, and a body naming no repository is unreadable. del := func(refType, ref string) []byte { - b, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-sdk", "full_name": "novox/mesh-sdk", "ref": ref, "ref_type": refType, "sha": head}) + b, _ := json.Marshal(map[string]any{"repository": "novox/mesh-sdk", "ref": ref, "ref-type": refType, "sender": "jochen"}) return b } msg, err := Convert(DeletedSubject, del("branch", "feat/x"), nil, anyFiles) var bd BranchDeleted if err != nil || msg.Event != "forge.novox.mesh-sdk.branch-deleted" || json.Unmarshal(msg.Data, &bd) != nil || - bd.Kind != ForgeBranchDeleted || bd.Repository != "novox/mesh-sdk" || bd.Branch != "feat/x" || bd.Head != head || msg.Said == "" { + bd.Kind != ForgeBranchDeleted || bd.Repository != "novox/mesh-sdk" || bd.Branch != "feat/x" || bd.Head != "" || msg.Said == "" { t.Errorf("forge.delete of a branch = %+v (%s), %v", msg, msg.Data, err) } for _, c := range []struct{ refType, ref string }{{"tag", "go/v0.1.13"}, {"branch", "main"}} { @@ -193,7 +213,7 @@ func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236 t.Errorf("forge.delete of %s %s: err = %v, want ErrNotHandled", c.refType, c.ref, err) } } - if _, err := Convert(DeletedSubject, []byte(`{"ref":"x","ref_type":"branch"}`), nil, anyFiles); !errors.Is(err, ErrUnreadable) { + if _, err := Convert(DeletedSubject, []byte(`{"ref":"x","ref-type":"branch"}`), nil, anyFiles); !errors.Is(err, ErrUnreadable) { t.Errorf("forge.delete naming no repository: err = %v, want ErrUnreadable", err) } // 1236: a merged event without a full merge commit is unreadable, and the forge is not asked. @@ -206,27 +226,13 @@ func TestOtherSubjectsAreNotHandled(t *testing.T) { // 1235, 1236 const merge = "89abcdef0123456789abcdef0123456789abcdef" -// mergedBody is gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 writes it. +// mergedBody is the seat git's pull.merged, extra by the short names seatPull reads. func mergedBody(extra map[string]any) []byte { - m := map[string]any{ - "owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "A change", "body": "text", - "head_sha": head, "head_checks": map[string]string{"mesh/merge-gate": "success"}, "head": "feat/x", "base": "main", - "merge_commit_sha": merge, "merged_at": "2026-10-19T12:00:00Z", "clone_url": "https://forge.example/novox/mesh-delivery.git", - "html_url": "https://forge.example/novox/mesh-delivery/pulls/12", "paths": []string{"modules/a/x.go"}, "paths_truncated": false, - "removed": []string{}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true, - } - for k, v := range extra { - if v == nil { - delete(m, k) - } else { - m[k] = v - } - } - b, _ := json.Marshal(m) - return b + return seatPull(map[string]any{"merge_commit_sha": merge, "merged_at": "2026-10-19T12:00:00Z", + "paths": []string{"modules/a/x.go"}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true}, extra) } -const mergedSubject = "mesh.mod.gitea.event.pull.merged" +const mergedSubject = "mesh.seat.git.event.pull-merged" type call struct{ owner, repo, sha string } @@ -311,16 +317,15 @@ func TestNoModuleJSONAnywhereWritesAMergedMessageWithNoTrees(t *testing.T) { } } -// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds each entry -// (owner, repo, number, base, head, head_sha, html_url, required, statuses, checks, labels) inside {pulls}. +// openAnswer is the seat git's answer of open-pulls. func openAnswer(pulls ...map[string]any) []byte { b, _ := json.Marshal(map[string]any{"pulls": pulls}) return b } func openEntry(repo string, number int, base, branch, sha string) map[string]any { - return map[string]any{"owner": "novox", "repo": repo, "number": number, "base": base, "head": branch, "head_sha": sha, - "html_url": "https://forge.example/x", "required": []string{"mesh/merge-gate"}, "statuses": []string{}, "checks": map[string]string{}, "labels": []string{}} + return map[string]any{"pull": map[string]any{"repository": "novox/" + repo, "number": number, "base": base, "branch": branch, "head": sha, + "state": "open", "url": "https://forge.example/x"}, "required": []string{"mesh/check"}, "checks": map[string]any{}} } func TestOpenPullsAreUpdatedPullMessagesWithUnknownDirs(t *testing.T) { // 1237/AC1 @@ -355,7 +360,7 @@ func TestOpenPullsUnreadableEntryIsNamedAndTheRestGo(t *testing.T) { // 1237, 12 if err != nil || len(msgs) != 1 || len(skipped) != 1 || !strings.Contains(skipped[0], "novox/a#1") { t.Errorf("msgs %d skipped %v err %v", len(msgs), skipped, err) } - for _, bad := range []string{`{"pulls":null}`, `{}`, `[]`, `x`} { + for _, bad := range []string{`{"pulls":null}`, `{}`, `[]`, `x`, string(gitFixture(t, "open-pulls answers an error, never none", "verb:open-pulls.answer"))} { if _, _, err := OpenPulls([]byte(bad)); err == nil { t.Errorf("%s was taken as an answer", bad) } @@ -381,7 +386,7 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6 if f, derr := forgemsg.DecodeForge(m.Data); err != nil || derr != nil || f.Kind != forgemsg.ForgeOpen || len(f.Open) != 0 { t.Errorf("empty answer: %+v %v %v", f, err, derr) } - for _, bad := range []string{`{"pulls":null}`, `{}`, `x`} { + for _, bad := range []string{`{"pulls":null}`, `{}`, `x`, `{"error":"the forge answered 502","pulls":[]}`} { if _, err := OpenMessage([]byte(bad)); err == nil { t.Errorf("%s was taken as an answer", bad) } @@ -411,41 +416,66 @@ func TestTrackSaysWhichHeadOfWhichPullRequestWasWritten(t *testing.T) { // 1279/ } } -func TestBridgeTagsAnswersAPackagesTagsByPrefixPageByPage(t *testing.T) { // DLV-FB1, design 51 *The floor* - var asked []string - page := func(n int) json.RawMessage { - var tags []map[string]string - for i := 0; i < n; i++ { - tags = append(tags, map[string]string{"name": "x" + strings.Repeat("y", i)}) +// gitFixture is one answer of the seat git's fixtures, by its name and what it is of. +func gitFixture(t *testing.T, name, of string) json.RawMessage { + t.Helper() + return gitfixture.Given(t, name, of) +} + +func TestBridgeTagsAnswersAPackagesTagsByPrefixPageByPage(t *testing.T) { // DLV-FB1, design 51 *The floor*; design 47 *The seat git* + more := gitFixture(t, "tags answers", "verb:tags.answer") // go/v0.1.15, go/v0.1.14, more + failed := gitFixture(t, "tags answers an error, never none", "verb:tags.answer") + last := json.RawMessage(`{"repository":"novox/mesh-sdk","tags":["go/v0.1.2","v0.0.1"],"page":2,"bound":50}`) + type asked struct { + key string + in string + } + for _, row := range []struct { + name string + answer func(page int) (json.RawMessage, error) + repo string + tags []string + more bool + pages int + failing string + }{ + {name: "two pages, the second says no more", repo: "novox/mesh-sdk", tags: []string{"go/v0.1.14", "go/v0.1.15", "go/v0.1.2"}, pages: 2, + answer: func(p int) (json.RawMessage, error) { + if p == 1 { + return more, nil + } + return last, nil + }}, + {name: "every page says more: stopped at the bound", repo: "novox/mesh-sdk", more: true, pages: TagsMaxPages, + answer: func(int) (json.RawMessage, error) { return json.RawMessage(`{"tags":[],"more":true}`), nil }}, + {name: "the seat does not answer", repo: "novox/mesh-sdk", failing: "page 1", pages: 1, + answer: func(int) (json.RawMessage, error) { return nil, errors.New("down") }}, + {name: "an error answer is an error, never none", repo: "novox/mesh-sdk", failing: "404", pages: 1, + answer: func(int) (json.RawMessage, error) { return failed, nil }}, + {name: "a repository with no owner", repo: "mesh-sdk", failing: "/"}, + } { + var calls []asked + call := func(_ context.Context, key string, body any) (json.RawMessage, error) { + in, _ := json.Marshal(body) + calls = append(calls, asked{key, string(in)}) + var p struct{ Page int } + _ = json.Unmarshal(in, &p) + return row.answer(p.Page) } - b, _ := json.Marshal(tags) - return b - } - api := func(path string) (json.RawMessage, error) { - asked = append(asked, path) - if strings.Contains(path, "page=1&") { - full := page(TagsPageSize - 2) - var tags []map[string]string - _ = json.Unmarshal(full, &tags) - tags = append(tags, map[string]string{"name": "go/v0.1.13"}, map[string]string{"name": "v0.0.1"}) - b, _ := json.Marshal(map[string]any{"result": tags}) - return b, nil + a, err := Tags(context.Background(), call, row.repo, "go/v") + if row.failing != "" { + if err == nil || !strings.Contains(err.Error(), row.failing) { + t.Errorf("%s: %v, want an error naming %q", row.name, err, row.failing) + } + } else if err != nil || !slicesEqual(a.Tags, row.tags) || a.More != row.more { + t.Errorf("%s: Tags = %+v, %v", row.name, a, err) + } + if len(calls) != row.pages { + t.Errorf("%s: %d calls, want %d", row.name, len(calls), row.pages) + } + if len(calls) > 0 && (calls[0].key != "seat:git.tags" || calls[0].in != `{"page":1,"prefix":"go/v","repository":"novox/mesh-sdk"}`) { + t.Errorf("%s: first call %+v", row.name, calls[0]) } - return json.RawMessage(`{"result":[{"name":"go/v0.1.2"}]}`), nil - } - a, err := Tags(api, "novox/mesh-sdk", "go/v") - if err != nil || !slicesEqual(a.Tags, []string{"go/v0.1.13", "go/v0.1.2"}) || a.More || len(asked) != 2 || - asked[0] != "/repos/novox/mesh-sdk/tags?page=1&limit=50" { - t.Errorf("Tags = %+v, %v; asked %v", a, err, asked) - } - if _, err := Tags(func(string) (json.RawMessage, error) { return nil, errors.New("down") }, "novox/mesh-sdk", ""); err == nil || !strings.Contains(err.Error(), "page 1") { - t.Errorf("a forge not answering = %v, want an error naming the page", err) - } - if _, err := Tags(func(string) (json.RawMessage, error) { return json.RawMessage(`[{"name":"go/v0.1.2"}]`), nil }, "novox/mesh-sdk", ""); err == nil || !strings.Contains(err.Error(), `{"result"`) { - t.Errorf("a bare list with no wrapper = %v, want the shape named", err) - } - if _, err := Tags(api, "mesh-sdk", ""); err == nil { - t.Error("a repository with no owner answered") } } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go index d1160c79..344cc9bd 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go @@ -3,77 +3,106 @@ package bridge import ( "context" "encoding/json" + "errors" "fmt" - "regexp" + "strings" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" ) -// Caller calls a module's tool by its manifest key (`.`) with a JSON body and returns the tool's result: -// the bridge's process calls through the bus as its own principal, its tools bundle through the runtime (stdio.Ask). +// Caller calls a seat's verb by its manifest key (`seat:.`) with a JSON body and returns its answer: the +// bridge's process calls through the bus as its own principal, its tools bundle through the runtime (stdio.Ask). type Caller func(ctx context.Context, key string, body any) (json.RawMessage, error) -// CommitTreesTool is the forge's tool for a commit's trees (mesh-catalog modules/gitea, issue 1234): arguments owner, -// repo and sha, answer {root, trees}, the root directory spelled "" (1234/AC4). -const CommitTreesTool = "gitea.gitea_commit_trees" +// GitKey is the manifest key of a verb of the seat `git` (novox/hq design 47 *The seat `git`*): the bridge reaches the +// forge through the seat alone and names no module that holds it. +func GitKey(verb string) string { return "seat:" + git.Seat + "." + verb } -// TreesVia asks the forge for a commit's trees through call. +// askGit calls one verb of the seat `git` and decodes its answer into A. An answer the contract cannot type, or one +// that says an error, is the call's error: a failed read is never read as none (design 47 *A failed read answers an +// error*). +func askGit[A any](ctx context.Context, call Caller, verb string, in any) (A, error) { + var a A + raw, err := call(ctx, GitKey(verb), in) + if err != nil { + return a, fmt.Errorf("the seat git did not answer %s: %w", verb, err) + } + if err := json.Unmarshal(raw, &a); err != nil { + return a, fmt.Errorf("the seat git's answer of %s is not its contract's: %w", verb, err) + } + var said struct { + Error string `json:"error"` + } + _ = json.Unmarshal(raw, &said) + if said.Error != "" { + return a, fmt.Errorf("the seat git refused %s: %s", verb, said.Error) + } + return a, nil +} + +// repository is `/`, as the seat names a repository. +func repository(owner, repo string) string { return owner + "/" + repo } + +// TreesVia asks the seat `git` for a commit's trees (verb trees). func TreesVia(ctx context.Context, call Caller) Trees { return func(owner, repo, sha string) (string, map[string]string, error) { - raw, err := call(ctx, CommitTreesTool, map[string]string{"owner": owner, "repo": repo, "sha": sha}) + a, err := askGit[git.TreesAnswer](ctx, call, git.VerbTrees, git.TreesInput{Repository: repository(owner, repo), Commit: sha}) if err != nil { return "", nil, err } - var a struct { - Root string `json:"root"` - Trees map[string]string `json:"trees"` - } - if err := json.Unmarshal(raw, &a); err != nil { - return "", nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", CommitTreesTool, err) - } if a.Root == "" { - return "", nil, fmt.Errorf("the answer of %s names no root tree", CommitTreesTool) + return "", nil, fmt.Errorf("the seat git's trees of %s at %s name no root tree", repository(owner, repo), sha) } return a.Root, a.Trees, nil } } -// GetFileTool is the forge's tool for one file at a ref (mesh-catalog modules/gitea tools/index.ts gitea_get_file): -// arguments owner, repo, path and ref, answer {file: {path, ref, sha, size, content}}, the content decoded to text -// (client.ts getFile). A path the forge does not have fails with `Gitea API : 404 ` (client.ts request). -const GetFileTool = "gitea.gitea_get_file" - -// notFound is how the forge's error says 404 (client.ts request: `Gitea API ${path}: ${status} ${text}`; client.ts:670 -// recognises it as /: 404 /). -var notFound = regexp.MustCompile(`: 404 `) - -// FilesVia asks the forge for a file through call. A file the forge says is not there is ErrNotFound. +// FilesVia asks the seat `git` for one file at a ref (verb file). A file the seat says is absent is ErrNotFound; an +// answer that says neither absent nor content is an error, never an empty file. func FilesVia(ctx context.Context, call Caller) Files { return func(owner, repo, path, ref string) ([]byte, error) { - raw, err := call(ctx, GetFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref}) + raw, err := call(ctx, GitKey(git.VerbFile), git.FileInput{Repository: repository(owner, repo), Path: path, Ref: ref}) if err != nil { - if notFound.MatchString(err.Error()) { - return nil, fmt.Errorf("%w: %v", ErrNotFound, err) - } - return nil, err + return nil, fmt.Errorf("the seat git did not answer file: %w", err) } var a struct { - File *struct { - Content *string `json:"content"` - } `json:"file"` + git.FileAnswer + Content *string `json:"content"` } - if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil { - return nil, fmt.Errorf("the answer of %s is not {file: {content}}: %v", GetFileTool, err) + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the seat git's answer of file is not its contract's: %w", err) } - return []byte(*a.File.Content), nil + switch { + case a.Error != "": + return nil, fmt.Errorf("the seat git refused file %s at %s: %s", path, ref, a.Error) + case a.Absent: + return nil, fmt.Errorf("%w: %s at %s in %s", ErrNotFound, path, ref, repository(owner, repo)) + case a.Content == nil: + return nil, fmt.Errorf("the seat git's file %s at %s says neither its content nor that it is absent", path, ref) + } + return []byte(*a.Content), nil } } +// errNoAnswer is a read whose answer carries nothing the contract says it must. +var errNoAnswer = errors.New("the seat git's answer carries nothing") + // MergedMessage is the merged message of a merge into the trunk, asked again: built as the bridge builds the one it writes // on FORGE (it is that event's body through ConvertKept), from the pull request's owner, repository, number, branch, head // and merge commit, which the gate's merges state holds. It reads the forge for every module directory (no kept inputs). func MergedMessage(owner, repo string, number int, branch, head, merge string, trees Trees, files Files) (Message, error) { - body, err := json.Marshal(pull{Owner: owner, Repo: repo, Number: number, Head: branch, HeadSHA: head, Base: Trunk, MergeSHA: merge}) + body, err := json.Marshal(git.PullMergedEvent{Repository: repository(owner, repo), Number: int64(number), Branch: branch, Head: head, Base: Trunk, MergeCommit: merge}) if err != nil { return Message{}, err } return ConvertKept(MergedSubject, body, trees, files, nil) } + +// splitRepository is a `/` cut in two, refusing anything else. +func splitRepository(repository string) (string, string, error) { + owner, name, ok := strings.Cut(strings.TrimSpace(repository), "/") + if !ok || owner == "" || name == "" || strings.Contains(name, "/") { + return "", "", fmt.Errorf("repository %q is not /", repository) + } + return owner, name, nil +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go b/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go index 811504cc..022b74fb 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/forgemade.go @@ -1,10 +1,10 @@ package bridge import ( - "encoding/json" + "context" "fmt" - "net/url" - "strings" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" ) // ForgePull is a pull request as the forge itself says it: what the bridge needs of a merge made on the forge, not @@ -16,51 +16,30 @@ type ForgePull struct { Merge string // the merge commit into the trunk } -// ForgeMade reads, through the forge module's gitea_api (the path the bridge reads the forge by), the merged pull -// request whose merge commit is merge: by number when number is given (the pull request's merge commit must then equal -// merge), else by the merge commit (the forge's commit-to-pull-request lookup). A pull request not merged, merged into -// another branch than the trunk, or merged at another commit is refused. -func ForgeMade(api API, owner, repo string, number int, merge string) (ForgePull, error) { - path := fmt.Sprintf("/repos/%s/%s/commits/%s/pull", url.PathEscape(owner), url.PathEscape(repo), url.PathEscape(merge)) +// ForgeMade reads, through the seat `git`'s verb pull, the merged pull request whose merge commit is merge: by number +// when number is given (its merge commit must then equal merge), else by the merge commit. A pull request not merged, +// merged into another branch than the trunk, or merged at another commit is refused. +func ForgeMade(ctx context.Context, call Caller, owner, repo string, number int, merge string) (ForgePull, error) { + in := git.PullInput{Repository: repository(owner, repo), MergeCommit: merge} if number > 0 { - path = fmt.Sprintf("/repos/%s/%s/pulls/%d", url.PathEscape(owner), url.PathEscape(repo), number) + in = git.PullInput{Repository: repository(owner, repo), Number: int64(number)} } - raw, err := api(path) + a, err := askGit[git.PullAnswer](ctx, call, git.VerbPull, in) if err != nil { - return ForgePull{}, fmt.Errorf("the forge did not answer %s: %w", path, err) - } - body, err := forgeBody(raw) - if err != nil { - return ForgePull{}, fmt.Errorf("the forge's answer for %s: %w", path, err) - } - var p struct { - Number int `json:"number"` - Merged bool `json:"merged"` - MergeCommitSHA string `json:"merge_commit_sha"` - Head struct { - Label string `json:"label"` - SHA string `json:"sha"` - } `json:"head"` - Base struct { - Ref string `json:"ref"` - } `json:"base"` - } - if err := json.Unmarshal(body, &p); err != nil { - return ForgePull{}, fmt.Errorf("the forge's answer for %s is no pull request: %w", path, err) + return ForgePull{}, err } + p := a.Pull switch { - case p.Number == 0: - return ForgePull{}, fmt.Errorf("the forge's answer for %s is no pull request (no number)", path) - case !p.Merged: - return ForgePull{}, fmt.Errorf("the forge says the pull request %s/%s#%d is not merged", owner, repo, p.Number) - case p.MergeCommitSHA != merge: - return ForgePull{}, fmt.Errorf("the forge says %s/%s#%d was merged at %s, not at %s", owner, repo, p.Number, p.MergeCommitSHA, merge) - case p.Base.Ref != Trunk: - return ForgePull{}, fmt.Errorf("the forge says %s/%s#%d was merged into %s, not the trunk %s", owner, repo, p.Number, p.Base.Ref, Trunk) + case p == nil || p.Number == 0: + return ForgePull{}, fmt.Errorf("the pull request of %s at %s: %w", in.Repository, merge, errNoAnswer) + case p.State != git.PullStateMerged: + return ForgePull{}, fmt.Errorf("the forge says the pull request %s#%d is %s, not merged", in.Repository, p.Number, p.State) + case p.MergeCommit != merge: + return ForgePull{}, fmt.Errorf("the forge says %s#%d was merged at %s, not at %s", in.Repository, p.Number, p.MergeCommit, merge) + case p.Base != Trunk: + return ForgePull{}, fmt.Errorf("the forge says %s#%d was merged into %s, not the trunk %s", in.Repository, p.Number, p.Base, Trunk) + case p.Branch == "": + return ForgePull{}, fmt.Errorf("the forge gives no head branch for %s#%d", in.Repository, p.Number) } - // For a merged pull request the forge's head.ref is "refs/pull/N/head"; the branch is in head.label (same-repository pull requests). - if p.Head.Label == "" || strings.HasPrefix(p.Head.Label, "refs/") { - return ForgePull{}, fmt.Errorf("the forge gives no head branch for %s/%s#%d (label %q)", owner, repo, p.Number, p.Head.Label) - } - return ForgePull{Number: p.Number, Branch: p.Head.Label, Head: p.Head.SHA, Merge: p.MergeCommitSHA}, nil + return ForgePull{Number: int(p.Number), Branch: p.Branch, Head: p.Head, Merge: p.MergeCommit}, nil } diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/tags.go b/modules/mesh-delivery-forge-bridge/internal/bridge/tags.go index b4a942ec..df9d4380 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/tags.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/tags.go @@ -1,39 +1,20 @@ package bridge import ( - "encoding/json" + "context" "fmt" - "net/url" "sort" "strings" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" ) // TagsTool is the forge bridge's tool bridge_tags (novox/hq design 51 *The floor*): a package's git tags, which the // builder's floor reads as one of its three sources. const TagsTool = "bridge_tags" -// TagsPageSize is how many tags one page of the forge's tags answers; TagsMaxPages bounds the read. -const ( - TagsPageSize = 50 - TagsMaxPages = 40 -) - -// API asks the forge one GET of its API (the forge module's gitea_api), answering what the tool answers: the body -// wrapped as {"result": }. Every reader of an API answer takes the body through forgeBody, the one place that -// knows that shape (issue 1966: two readers took the wrapper for the body). -type API func(path string) (json.RawMessage, error) - -// forgeBody is the body of the forge's answer: the tool's "result". An answer without one is refused, naming the shape, -// never read as an empty body. -func forgeBody(raw json.RawMessage) (json.RawMessage, error) { - var w struct { - Result json.RawMessage `json:"result"` - } - if err := json.Unmarshal(raw, &w); err != nil || len(w.Result) == 0 || string(w.Result) == "null" { - return nil, fmt.Errorf(`the forge module's answer is not {"result": }`) - } - return w.Result, nil -} +// TagsMaxPages bounds the read: at most this many pages of the seat `git`'s verb tags. +const TagsMaxPages = 40 // TagsAnswer is what bridge_tags answers. type TagsAnswer struct { @@ -45,39 +26,30 @@ type TagsAnswer struct { } // Tags is bridge_tags: every tag of repository (`/`) whose name starts with prefix (`go/v` for the Go -// package of mesh-sdk's `go` directory, `v` or empty for a package at the root), sorted. A forge that does not answer -// is an error naming the page: the builder then counts this source silent. -func Tags(api API, repository, prefix string) (TagsAnswer, error) { - owner, name, ok := strings.Cut(strings.TrimSpace(repository), "/") - if !ok || owner == "" || name == "" || strings.Contains(name, "/") { - return TagsAnswer{}, fmt.Errorf("repository %q is not /", repository) +// package of mesh-sdk's `go` directory, `v` or empty for a package at the root), sorted, read page by page through the +// seat `git`'s verb tags until a page says no more. A page the seat does not answer is an error naming it: the builder +// then counts this source silent. +func Tags(ctx context.Context, call Caller, repository, prefix string) (TagsAnswer, error) { + owner, name, err := splitRepository(repository) + if err != nil { + return TagsAnswer{}, err } out := TagsAnswer{Repository: owner + "/" + name, Prefix: prefix, Tags: []string{}} - for page := 1; ; page++ { + for page := int64(1); ; page++ { if page > TagsMaxPages { out.More = true break } - raw, err := api(fmt.Sprintf("/repos/%s/%s/tags?page=%d&limit=%d", url.PathEscape(owner), url.PathEscape(name), page, TagsPageSize)) + a, err := askGit[git.TagsAnswer](ctx, call, git.VerbTags, git.TagsInput{Repository: out.Repository, Prefix: prefix, Page: page}) if err != nil { - return TagsAnswer{}, fmt.Errorf("the forge did not answer the tags of %s (page %d): %w", out.Repository, page, err) + return TagsAnswer{}, fmt.Errorf("the tags of %s (page %d): %w", out.Repository, page, err) } - body, err := forgeBody(raw) - if err != nil { - return TagsAnswer{}, fmt.Errorf("the forge's tags of %s (page %d): %w", out.Repository, page, err) - } - var tags []struct { - Name string `json:"name"` - } - if err := json.Unmarshal(body, &tags); err != nil { - return TagsAnswer{}, fmt.Errorf("the forge's tags of %s (page %d) are no list of tags: %w", out.Repository, page, err) - } - for _, t := range tags { - if strings.HasPrefix(t.Name, prefix) { - out.Tags = append(out.Tags, t.Name) + for _, t := range a.Tags { + if strings.HasPrefix(t, prefix) { + out.Tags = append(out.Tags, t) } } - if len(tags) < TagsPageSize { + if !a.More { break } } diff --git a/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go b/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go new file mode 100644 index 00000000..f3e5f09d --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go @@ -0,0 +1,42 @@ +// Package gitfixture reads the seat git's own fixtures (contracts/seats/git/fixtures/verbs.json), so the bridge's unit +// fakes answer what the seat's holder answers and nothing the contract does not say (design 47 *The seat `git`*). +package gitfixture + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" +) + +// All is every fixture's body named name of of (such as "file answers", "verb:file.answer"), in the file's order; a +// name with none fails the test. +func All(t testing.TB, name, of string) []json.RawMessage { + t.Helper() + _, here, _, _ := runtime.Caller(0) + raw, err := os.ReadFile(filepath.Join(filepath.Dir(here), "../../../../contracts/seats/git/fixtures/verbs.json")) + if err != nil { + t.Fatal(err) + } + var all []struct { + Name, Of string + Given json.RawMessage + } + if err := json.Unmarshal(raw, &all); err != nil { + t.Fatal(err) + } + var out []json.RawMessage + for _, f := range all { + if f.Name == name && f.Of == of { + out = append(out, f.Given) + } + } + if len(out) == 0 { + t.Fatalf("no fixture %q of %s", name, of) + } + return out +} + +// Given is the first fixture's body named name of of. +func Given(t testing.TB, name, of string) json.RawMessage { t.Helper(); return All(t, name, of)[0] } diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 0f4b8860..7a786eaa 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -91,15 +91,16 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 t.Errorf("streams = %+v", m.Streams) } // The checker's owed is the forge module's to write (ADR 0521, R4e finding R2-1): the bridge neither hears nor reads it. - // gitea.forge.delete: a branch's end, said as a branch-deleted kind (design 51, DLV-FB1). - if want := []string{"gitea.forge.delete", "gitea.pull.closed", "gitea.pull.merged", "gitea.pull.updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + // The forge's events are the seat git's (design 47 *The seat git*); forge.delete: a branch's end, said as a + // branch-deleted kind (design 51, DLV-FB1). + if want := []string{"git.forge-delete", "git.pull-closed", "git.pull-merged", "git.pull-updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if want := []string{"mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"gitea.gitea_add_comment", "gitea.gitea_api", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view", - "gitea.gitea_get_file", "gitea.gitea_merge_pull_request", "gitea.gitea_note_append", "gitea.gitea_open_pulls"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_delivery_view", "gitea.gitea_merge_pull_request", + "gitea.gitea_note_append", "seat:git.file", "seat:git.open-pulls", "seat:git.pull", "seat:git.tags", "seat:git.trees"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 2b214bd6..0105502f 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -3,10 +3,10 @@ "version": "1", "slug": "forge-bridge", "consumes": [ - "gitea.pull.updated", - "gitea.pull.merged", - "gitea.pull.closed", - "gitea.forge.delete" + "git.pull-updated", + "git.pull-merged", + "git.pull-closed", + "git.forge-delete" ], "streams": [ { "name": "forge", "events": ["forge.**"] } @@ -33,10 +33,11 @@ "gitea.gitea_delivery_view", "gitea.gitea_note_append", "gitea.gitea_add_comment", - "gitea.gitea_open_pulls", - "gitea.gitea_commit_trees", - "gitea.gitea_get_file", - "gitea.gitea_api" + "seat:git.open-pulls", + "seat:git.trees", + "seat:git.file", + "seat:git.pull", + "seat:git.tags" ], "own-secrets": { "broker": "${dir:state}/broker" From 799241364fea7d581c67cf9a7988c6f6192c5653 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:50:04 +0100 Subject: [PATCH 09/16] VW0: the view reads the trunk's source by the branch key and the per-branch value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §305 (the state source is kept per watched branch), §431 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-view/integration_test.go | 4 ++-- modules/mesh-delivery-view/internal/view/names.go | 5 +++++ modules/mesh-delivery-view/internal/view/reach.go | 2 +- modules/mesh-delivery-view/internal/view/reach_test.go | 6 +++--- modules/mesh-delivery-view/internal/view/tiers.go | 3 +-- modules/mesh-delivery-view/internal/view/view.go | 2 +- modules/mesh-delivery-view/internal/view/wants_test.go | 4 ++-- 7 files changed, 15 insertions(+), 11 deletions(-) diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go index cd326086..77ad3d76 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view/integration_test.go @@ -797,7 +797,7 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { want, _ := meshwantsclient.EncodeWant(meshwantsclient.Want{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64)}) failed, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed}) good, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) - src, _ := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) + src, _ := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "feat/x", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) for bucket, kv := range map[string][]byte{view.ModulesBucket: mod} { if _, err := kvs[bucket].Put(ctx, "web", kv); err != nil { t.Fatal(err) @@ -809,7 +809,7 @@ func TestProcessWholeSaysTheSourceRevisionIsDelivered(t *testing.T) { if _, err := kvs[view.HaveBucket].Put(ctx, "ace.web", failed); err != nil { t.Fatal(err) } - rev, err := kvs[view.SourceBucket].Put(ctx, meshdeliveryclient.SourceMainKey, src) + rev, err := kvs[view.SourceBucket].Put(ctx, meshdeliveryclient.SourceKey("main"), src) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-view/internal/view/names.go b/modules/mesh-delivery-view/internal/view/names.go index 0fc71ca2..4a2d62f1 100644 --- a/modules/mesh-delivery-view/internal/view/names.go +++ b/modules/mesh-delivery-view/internal/view/names.go @@ -3,6 +3,7 @@ package view import ( "fmt" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" @@ -88,3 +89,7 @@ func PackagesKey(module string) (string, error) { } return meshcatalog.PackagesByModuleKey(keys.EscapeName(module)), nil } + +// trunkSourceKey is the key of the gate's state source for the trunk, the watched branch `main` (design 47 G §305: the source +// is kept per watched branch, meshdeliveryclient.SourceKey; the view's reads are of the trunk's). +var trunkSourceKey = meshdeliveryclient.SourceKey("main") diff --git a/modules/mesh-delivery-view/internal/view/reach.go b/modules/mesh-delivery-view/internal/view/reach.go index b11e0648..d74e058c 100644 --- a/modules/mesh-delivery-view/internal/view/reach.go +++ b/modules/mesh-delivery-view/internal/view/reach.go @@ -71,7 +71,7 @@ func (m *Model) applyHistory(e Entry) { } m.modHist[e.Key] = h case SourceBucket: - if e.Key != meshdeliveryclient.SourceMainKey || e.Deleted || e.Revision == 0 { + if e.Key != trunkSourceKey || e.Deleted || e.Revision == 0 { return } src, err := meshdeliveryclient.DecodeSource(e.Value) diff --git a/modules/mesh-delivery-view/internal/view/reach_test.go b/modules/mesh-delivery-view/internal/view/reach_test.go index 12e32744..5d53d583 100644 --- a/modules/mesh-delivery-view/internal/view/reach_test.go +++ b/modules/mesh-delivery-view/internal/view/reach_test.go @@ -49,11 +49,11 @@ func TestTheViewDerivesHowFarEachRepositoryReached(t *testing.T) { return append([]Entry{modIn(fpLater, commit, 12, to, true), modIn(fpLater, commit, 12, to, false)}, extra...) } src := func(rev uint64, webHead string) Entry { - v, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{web: webHead, sdk: commitS}}) + v, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "g", Heads: map[string]string{web: webHead, sdk: commitS}}) if err != nil { t.Fatal(err) } - return Entry{Bucket: SourceBucket, Key: meshdeliveryclient.SourceMainKey, Value: v, At: t0, Revision: rev, History: true} + return Entry{Bucket: SourceBucket, Key: trunkSourceKey, Value: v, At: t0, Revision: rev, History: true} } taken := func(repository, merge string) Entry { key, _ := trunk.RepositoryKey(repository) @@ -72,7 +72,7 @@ func TestTheViewDerivesHowFarEachRepositoryReached(t *testing.T) { // web changed at A (fpOld) and at C (fpNew); mail first appeared at A. The source named A, B, then C as web's head. world := func(t *testing.T) []Entry { cur, _ := meshdeliveryclient.DecodeSource(src(12, commitC).Value) - current := Entry{Bucket: SourceBucket, Key: meshdeliveryclient.SourceMainKey, Revision: 12, At: t0} + current := Entry{Bucket: SourceBucket, Key: trunkSourceKey, Revision: 12, At: t0} current.Value, _ = meshdeliveryclient.EncodeSource(cur) return []Entry{src(10, commitA), src(11, commitB), src(12, commitC), current, mod("web", fpOld, commitA, 10, true), modIn(fpNew, commitC, 12, in("t1", "mesh-sdk/go"), true), mod("mail", fpMail, commitA, 0, true), diff --git a/modules/mesh-delivery-view/internal/view/tiers.go b/modules/mesh-delivery-view/internal/view/tiers.go index a579c044..1d720ef5 100644 --- a/modules/mesh-delivery-view/internal/view/tiers.go +++ b/modules/mesh-delivery-view/internal/view/tiers.go @@ -13,7 +13,6 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshwatchdogsclient" "git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" @@ -83,7 +82,7 @@ func (r Reader) Tiers(revision uint64) (Tiers, error) { if r.SourceState.GetEntry == nil { return Tiers{}, fmt.Errorf("the gate's source cannot be read here: name the source revision") } - _, rev, err := r.SourceState.GetEntry(meshdeliveryclient.SourceMainKey) + _, rev, err := r.SourceState.GetEntry(trunkSourceKey) if err != nil { return Tiers{}, fmt.Errorf("the gate's source: %w", err) } diff --git a/modules/mesh-delivery-view/internal/view/view.go b/modules/mesh-delivery-view/internal/view/view.go index 35f1c077..49db0b8e 100644 --- a/modules/mesh-delivery-view/internal/view/view.go +++ b/modules/mesh-delivery-view/internal/view/view.go @@ -621,7 +621,7 @@ func (m *Model) apply(e Entry) { delete(m.taken, e.Key) } case SourceBucket: - if e.Key != meshdeliveryclient.SourceMainKey { + if e.Key != trunkSourceKey { return } if e.Deleted { diff --git a/modules/mesh-delivery-view/internal/view/wants_test.go b/modules/mesh-delivery-view/internal/view/wants_test.go index 6599683e..23a84f33 100644 --- a/modules/mesh-delivery-view/internal/view/wants_test.go +++ b/modules/mesh-delivery-view/internal/view/wants_test.go @@ -61,11 +61,11 @@ func moduleE(t *testing.T, module, fp string) Entry { func sourceE(t *testing.T, revision uint64) Entry { t.Helper() - v, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByMainValue{Group: "g", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) + v, err := meshdeliveryclient.EncodeSource(meshdelivery.SourceByBranchValue{Group: "g", Heads: map[string]string{"novox/web": strings.Repeat("1", 40)}}) if err != nil { t.Fatal(err) } - return Entry{Bucket: SourceBucket, Key: meshdeliveryclient.SourceMainKey, Value: v, At: t0, Revision: revision} + return Entry{Bucket: SourceBucket, Key: trunkSourceKey, Value: v, At: t0, Revision: revision} } // 1906/D7 AC7: every want is a row beside the have of its node, with where it stands. From e7bd54e67bda305fb2cb5c3513f25dccfe4b79a1 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:51:07 +0100 Subject: [PATCH 10/16] IX19: the indexer reads every contracts module's trunk definitions at its start before judging, waits for the seat git at most two minutes, says view-waits past five and judges again once it answers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Taken from the join base (6ce86c96, ff6e74f9) onto the build branch. Builds: design 47 G §198, §248, §316, §255, §57 (tests 54, 57) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-indexer/main.go | 28 ++- .../cmd/mesh-delivery-indexer/views.go | 25 +++ .../internal/index/control_records_test.go | 59 +++++++ .../internal/index/seatdefs.go | 46 +++++ .../internal/index/startread.go | 162 ++++++++++++++++++ 5 files changed, 315 insertions(+), 5 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/startread.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 01e3af36..8bb6284f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -591,6 +591,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := ix.Reconcile(ctx); err != nil { logger.Printf("dropping the modules the trunk does not hold at start: %v", err) } + // Every contracts module's trunk definitions read before any manifest is judged and before the forge bridge's stream is + // followed (design 47, the indexer reads every definition before it judges): a definition merged while it was down, or + // one no event announced, is never missing from the judging below. Once the state is bound, so on its first binding too. + // Through the seat git (milestone 47/G W4.15, design 47 *A part that cannot read the forge keeps what it has*): a seat that + // does not answer is waited for at most index.GitWaitBound before the judging below runs on the kept definitions, said in + // indexer_first_views and as view-waits past five minutes, and read again on the next forge message heard. + defs := gitWaits(ctx, ix, findings, process.AwaitBucket(ctx, js, conn, firstViewsBucket, "indexer_first_views does not say the wait for the seat git", logger.Printf), logger.Printf) + seatDefs.Ready(func(jetstream.KeyValue) { defs.Start(ctx) }) // Every manifest judged at the start, so a judge that changed with this program judges every one again (ADR 0578 // §1), once the verdicts state is there; each merge judges them again (index.ApplyMessage). verdicts.Ready(func(jetstream.KeyValue) { @@ -603,13 +611,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }() // A pass that fails (a read the bus did not answer) is judged again at the next message or source revision the // indexer hears (index.kickJudge): no timer. - j, err := ix.JudgeAtStart(ctx) - if err != nil { - logger.Printf("judging the manifests at start: %v; judged again at the next message heard", err) + judge := func(fromKept bool) { + j, err := ix.JudgeAtStart(ctx) + if err != nil { + logger.Printf("judging the manifests at start (from kept definitions: %v): %v; judged again at the next message heard", fromKept, err) + return + } + logger.Printf("judged %d manifests at start (from kept definitions: %v): %d verdicts written, %d with problems, of which outgrown and kept accepted: %v, waiting for a judge that knows their fields: %v", + j.Judged, fromKept, j.Written, len(j.Refused), j.Outgrown, j.Waiting) + } + if !defs.Waiting() { + judge(false) return } - logger.Printf("judged %d manifests at start: %d verdicts written, %d with problems, of which outgrown and kept accepted: %v, waiting for a judge that knows their fields: %v", - j.Judged, j.Written, len(j.Refused), j.Outgrown, j.Waiting) + // The seat git did not answer: the judging waits for it at most index.GitWaitBound, off the start, so nothing else of + // the start is held behind it. + go func() { judge(defs.Ready(ctx)) }() }) // A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are // made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again @@ -736,6 +753,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro handle(ctx, m, ix, heard, logger) ix.KickFill(ctx) unreadAgain(ctx, ix, logger) + defs.Heard(ctx) }, logger.Printf) if err != nil { return err diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go index 89d56bdf..a1afd067 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/views.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" "sync/atomic" "time" @@ -180,3 +181,27 @@ func followAssignments(ctx context.Context, hub *process.EventHub, cred process. } return f } + +// gitWaits is the indexer's read of the trunk definitions at its start through the seat git (index.DefinitionsAtStart): its +// wait said as a first view of the seat's (Owner git, State seat-definitions) in the indexer's first views, and its finding +// view-waits written past views.WaitsBound and cleared once the seat answers. +func gitWaits(ctx context.Context, ix *index.Indexer, findings, status *process.AwaitedBucket, logf func(string, ...any)) *index.DefinitionsAtStart { + finding := func(s views.Status, raised bool) { + w := sdkfindings.Writer(ctx, findings.KV) + f := views.ViewWaitsFinding(index.Module, s, time.Now()) + var err error + if raised { + err = w.Put(f) + } else { + err = w.Delete(f) + } + if err != nil { + logf("views: the finding view-waits of the seat git is not written: %v", err) + } + } + f := &views.Follower{Owner: git.Seat, Tool: "seat:" + git.Seat + "." + git.VerbTrees, State: index.SeatDefinitionsState, Logf: logf, + RaiseWaits: func(s views.Status) { finding(s, true) }, ClearWaits: func(s views.Status) { finding(s, false) }} + status.Ready(func(kv jetstream.KeyValue) { f.Said = views.KeepStatus(ctx, kv, logf) }) + return &index.DefinitionsAtStart{Read: ix.ReadTrunkDefinitions, Waits: f, Logf: logf, + Judge: func(ctx context.Context) error { _, err := ix.JudgeAtStart(ctx); return err }} +} diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go index 66f7948c..337fb881 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records_test.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -125,6 +125,36 @@ func TestTheControlRecords(t *testing.T) { // and source, says seat-definition-changed only when a digest moved, keeps the trunk's entry over one that does not // parse (its finding), removes a pull request's entries when it closes, keeps a verdict's out-of-step files, and // raises generated-types-out-of-step from a trunk build end, cleared only by a clean one of the same module. + // Milestone 47/G W4.15, design 47 test 54: the seat git not answering at start holds the judging at most its bound, which + // then runs on the kept definitions with the wait said; the read once the forge is heard holds the view and judges again. + t.Run("W4.15 the start waits for the seat git, judges from kept definitions, and judges again once it answers", func(t *testing.T) { + answers := []error{fmt.Errorf("mesh-contracts: %w", errForgeRead), fmt.Errorf("again: %w", errForgeRead), nil} + var said []string + judged := 0 + d := &DefinitionsAtStart{Bound: time.Millisecond, Waits: waitsFake{&said}, + Read: func(context.Context) (int, error) { err := answers[0]; answers = answers[1:]; return 1, err }, + Judge: func(context.Context) error { judged++; return nil }} + ctx := context.Background() + d.Start(ctx) + if !d.Waiting() || len(said) != 1 || said[0] != "waits "+GitDefinitionsView { + t.Fatalf("waiting %v, said %q: want the wait for the seat git said once", d.Waiting(), said) + } + if !d.Ready(ctx) || judged != 0 { + t.Fatalf("the start past its bound: want it judged from the kept definitions, judged again %d", judged) + } + d.Heard(ctx) // the forge heard, the seat still not answering: the wait stands, said once + if !d.Waiting() || len(said) != 1 || judged != 0 { + t.Fatalf("a read still not answered: waiting %v, said %q, judged %d", d.Waiting(), said, judged) + } + d.Heard(ctx) + if d.Waiting() || len(said) != 2 || said[1] != "holds "+GitDefinitionsView || judged != 1 || d.Ready(ctx) { + t.Errorf("the seat answering: waiting %v, said %q, judged again %d: want the view held and every manifest judged again once", d.Waiting(), said, judged) + } + d.Heard(ctx) // nothing waits: no read, no judging + if judged != 1 || len(answers) != 0 { + t.Errorf("a forge message with nothing waiting read or judged again") + } + }) t.Run("seat definitions", func(t *testing.T) { const repo, m1, m2, m3, head = "novox/mesh-contracts", "m1", "m2", "m3", "h1" manifest := `{"module":"mesh-contracts","version":"1.0","build":{"artifacts":[{"name":"go","kind":"package","language":"go","paths":["go"],"series":"0.1","generated-from":"."}]}}` @@ -315,6 +345,29 @@ func TestTheControlRecords(t *testing.T) { t.Errorf("said %+v: only a trunk's definition is said (ADR 0636 §4)", c) } } + // Design 47 (the indexer reads every definition before it judges): at its start the indexer reads every contracts + // module's trunk at its repository's head in the source revision, so a definition merged with no event heard is kept + // before any manifest is judged; a repository the source names no head of is read at the module's own commit. + kept, _ = meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mesh-contracts", Manifest: manifest}) + mkey, _ = meshcatalogclient.ManifestsKey(meshcatalogclient.ManifestDigest([]byte(manifest))) + ix.Manifests.(*memStore).vals[mkey] = kept + mod, _ = meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: "src2:" + strings.Repeat("a", 64), Manifest: meshcatalogclient.ManifestDigest([]byte(manifest)), Repository: repo, Commit: m3, Path: meshcatalogclient.RootPath}) + ix.Modules.(*memStore).vals["mesh-contracts"] = mod + files["h3"] = map[string]string{"seats/git/seat.json": strings.Replace(git, "ADR 0111", "ADR 0500", 1)} + ix.source = sourceNow{rev: 7, heads: map[string]string{repo: "h3"}} + if n, err := ix.ReadTrunkDefinitions(ctx); err != nil || n != 1 { + t.Fatalf("read at start = %d, %v; want the one contracts module", n, err) + } + if e, _ := entry("git.trunk"); e.Commit != "h3" { + t.Errorf("git.trunk after the read at start = %+v, want the source's head h3", e) + } + ix.source = sourceNow{} + if _, err := ix.ReadTrunkDefinitions(ctx); err != nil { + t.Fatal(err) + } + if e, _ := entry("git.trunk"); e.Commit != m3 { + t.Errorf("git.trunk read with no source head = %+v, want the module's own commit", e) + } // ADR 0636 §5, D-I3, D-I5: a seat of the mesh's own contracts is taken only from its domain's contracts module; a // move there is a handover in either order of the two trunks; a second definer is refused with a finding naming both. @@ -636,3 +689,9 @@ func (m memSource) Keys() ([]string, error) { } func (m memSource) Get(key string) (*views.Entry, bool, error) { e, ok := m[key]; return e, ok, nil } + +// waitsFake records what the indexer says it waits for and holds. +type waitsFake struct{ said *[]string } + +func (w waitsFake) WaitFor(_ context.Context, view string) { *w.said = append(*w.said, "waits "+view) } +func (w waitsFake) Holds(view string) { *w.said = append(*w.said, "holds "+view) } diff --git a/modules/mesh-delivery-indexer/internal/index/seatdefs.go b/modules/mesh-delivery-indexer/internal/index/seatdefs.go index 87d3125e..488bb37f 100644 --- a/modules/mesh-delivery-indexer/internal/index/seatdefs.go +++ b/modules/mesh-delivery-indexer/internal/index/seatdefs.go @@ -13,12 +13,14 @@ package index // contracts module. import ( + "cmp" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" + "maps" "path" "slices" "strconv" @@ -30,6 +32,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/sdk/seatdefs" + "git.novox.be/novox/mesh-delivery/sdk/trunk" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/seatdef" "git.novox.be/novox/mesh-sdk/go/views" @@ -278,6 +281,49 @@ func (ix *Indexer) SeatDefinitionsAgain(ctx context.Context, module string, pull return out } +// ReadTrunkDefinitions reads, at the start and before any manifest is judged, the seat definitions of every contracts +// module the trunk holds, at its repository's head in the source revision (at the module's own commit where the source +// names none), and keeps them as a merge does (design 47, *The indexer reads every definition before it judges*): a +// definition merged while the indexer was down, or one no event announced, is kept before a manifest claiming its seat +// is judged. It answers how many contracts modules were read; one the forge did not answer is in the error, and is read +// again at its next merged message or by indexer_seat_definitions_again. +func (ix *Indexer) ReadTrunkDefinitions(ctx context.Context) (int, error) { + if ix.SeatDefinitions == nil || ix.ReadFile == nil || ix.Modules == nil || ix.Manifests == nil { + return 0, nil + } + held, err := ix.heldModules(ctx) + if err != nil { + return 0, err + } + ix.mu.Lock() + heads := maps.Clone(ix.source.heads) + ix.mu.Unlock() + read := 0 + var errs []error + for _, name := range slices.Sorted(maps.Keys(held)) { + m := held[name] + manifest, _, err := ix.manifestRaw(ctx, m.Manifest) + if err != nil { + errs = append(errs, fmt.Errorf("%s's manifest: %w", name, err)) + continue + } + dir := cmp.Or(m.Path, meshcatalogclient.RootPath) + commit := m.Commit + for r, head := range heads { + if trunk.SameRepository(r, m.Repository) && head != "" { + commit = head + } + } + for _, c := range contractsModules(map[string]string{dir: manifest}) { + read++ + if err := ix.keepTrunk(ctx, m.Repository, commit, c); err != nil { + errs = append(errs, fmt.Errorf("%s at %.12s: %w", name, commit, err)) + } + } + } + return read, errors.Join(errs...) +} + // keepTrunk writes a contracts module's trunk definitions at a merge: each that parses where its digest moved (said), // each the module's index no longer names removed (said with no digest), and the module's unparsed finding written or // cleared. A definition that does not parse leaves the trunk's entry as it stands. diff --git a/modules/mesh-delivery-indexer/internal/index/startread.go b/modules/mesh-delivery-indexer/internal/index/startread.go new file mode 100644 index 00000000..feea611a --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/startread.go @@ -0,0 +1,162 @@ +package index + +import ( + "context" + "errors" + "sync" + "time" +) + +// GitWaitBound is how long the indexer waits, at its start, for the seat git to answer the read of every contracts module's +// trunk definitions before it judges from the definitions it kept (design 47, *A part that cannot read the forge keeps +// what it has, and says so*): longer than a forge's restart, short enough that a start is not held behind a forge down +// for good. +const GitWaitBound = 2 * time.Minute + +// GitDefinitionsView is what the indexer says it waits for in its first views while the seat git does not answer. +const GitDefinitionsView = "trunk seat definitions" + +// ViewWaiter is where the indexer says it waits for the seat git and since when, and raises view-waits past its bound +// (mesh-sdk views.Follower's WaitFor and Holds). +type ViewWaiter interface { + WaitFor(ctx context.Context, view string) + Holds(view string) +} + +// DefinitionsAtStart is the indexer's read of the trunk definitions at its start through the seat git: the first read, +// a wait of at most Bound for the seat when it does not answer, the judging from the kept definitions after it, and the +// read again once the seat answers, which judges again every manifest judged from kept definitions. +type DefinitionsAtStart struct { + // Read reads every contracts module's trunk definitions (Indexer.ReadTrunkDefinitions). + Read func(ctx context.Context) (int, error) + // Judge judges every manifest again, as at the start (Indexer.JudgeAtStart). + Judge func(ctx context.Context) error + Waits ViewWaiter + Bound time.Duration + Logf func(string, ...any) + + mu sync.Mutex + answered chan struct{} // closed once a read is answered + waiting bool // a read was not answered and no read since was + fromKept bool // the judging at start ran on kept definitions +} + +func (d *DefinitionsAtStart) init() { + if d.answered == nil { + d.answered = make(chan struct{}) + } +} + +func (d *DefinitionsAtStart) logf(format string, args ...any) { + if d.Logf != nil { + d.Logf(format, args...) + } +} + +// Start reads the trunk definitions once; a seat that does not answer is said as a wait (Waits.WaitFor) and read again on +// the next sign that the forge runs (Heard). A read answered with other errors (a manifest it cannot read) is answered. +func (d *DefinitionsAtStart) Start(ctx context.Context) { + d.mu.Lock() + d.init() + d.mu.Unlock() + d.read(ctx, "at start") +} + +func (d *DefinitionsAtStart) read(ctx context.Context, when string) bool { + n, err := d.Read(ctx) + if errors.Is(err, errForgeRead) { + d.mu.Lock() + first := !d.waiting + d.waiting = true + d.mu.Unlock() + if first { + d.logf("the seat git did not answer the read of %d contracts modules' trunk definitions %s: %v; the indexer judges from the definitions it kept after %s, and reads them again once the seat answers", n, when, err, d.bound()) + if d.Waits != nil { + d.Waits.WaitFor(ctx, GitDefinitionsView) + } + } + return false + } + if err != nil { + d.logf("reading the seat definitions of %d contracts modules' trunks %s: %v; each is read again at its next merged message or by indexer_seat_definitions_again", n, when, err) + } else { + d.logf("read the seat definitions of %d contracts modules' trunks %s", n, when) + } + d.mu.Lock() + was := d.waiting + d.waiting = false + select { + case <-d.answered: + default: + close(d.answered) + } + again := d.fromKept + d.fromKept = false + d.mu.Unlock() + if was && d.Waits != nil { + d.Waits.Holds(GitDefinitionsView) + } + if again && d.Judge != nil { + if err := d.Judge(ctx); err != nil { + d.logf("judging again the manifests judged from kept definitions: %v; judged again at the next message heard", err) + } else { + d.logf("judged again every manifest judged from kept definitions, now that the seat git answers") + } + } + return true +} + +func (d *DefinitionsAtStart) bound() time.Duration { + if d.Bound > 0 { + return d.Bound + } + return GitWaitBound +} + +// Ready waits, while the seat git does not answer the read at start, until a read is answered, or Bound passed (no timer: a context's bound), and says whether the +// judging that follows runs on the definitions the indexer kept; it marks it so, and the read that answers later judges +// again (Read's answer). +func (d *DefinitionsAtStart) Ready(ctx context.Context) (fromKept bool) { + d.mu.Lock() + d.init() + answered, waiting := d.answered, d.waiting + d.mu.Unlock() + if !waiting { + return false // read, or not read at all (the seat definitions' state not bound yet): nothing to wait for + } + wait, cancel := context.WithTimeout(ctx, d.bound()) + defer cancel() + select { + case <-answered: + return false + case <-wait.Done(): + } + d.mu.Lock() + defer d.mu.Unlock() + select { + case <-d.answered: + return false + default: + } + d.fromKept = true + d.logf("the seat git has not answered within %s: the indexer judges from the definitions it kept", d.bound()) + return true +} + +// Heard is a sign that the forge runs (a message the forge bridge relays): while the indexer waits for the seat git, it +// reads the trunk definitions again. +func (d *DefinitionsAtStart) Heard(ctx context.Context) { + d.mu.Lock() + waiting := d.waiting + d.mu.Unlock() + if waiting { + d.read(ctx, "once the forge was heard") + } +} + +// Waiting says the indexer waits for the seat git. +func (d *DefinitionsAtStart) Waiting() bool { + d.mu.Lock() + defer d.mu.Unlock() + return d.waiting +} From 3b31e6df05087fcdb2e854d38dd4dc90ddbfe5fd Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:53:07 +0100 Subject: [PATCH 11/16] AS1: a want follows a passed pipeline, by compare-and-set per module and watched branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assigner hears pipeline-ended and group-ended, keeps the newest end per module and branch in the state written (tool assigner_written), and writes a want only from an end whose required steps passed. Builds: design 47 G §29, §35, §102, §120, §291, §325, §352, §85 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/main.go | 17 +- .../cmd/mesh-delivery-assigner/pass.go | 14 + .../cmd/mesh-delivery-assigner/pipelines.go | 327 ++++++++++++++++++ .../mesh-delivery-assigner/pipelines_test.go | 174 ++++++++++ .../cmd/mesh-delivery-assigner/tools.go | 1 + .../cmd/mesh-delivery-assigner/world.go | 14 + .../mesh-delivery-assigner/manifest_test.go | 6 +- modules/mesh-delivery-assigner/module.json | 8 +- 8 files changed, 554 insertions(+), 7 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 3ec09822..4f040bed 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -79,7 +79,7 @@ var readFamilies = []string{buckets.ComposedRead} var ownBuckets = []string{buckets.Asks, buckets.Differences, buckets.Puts, buckets.Wants, plc.AssignmentsBucket, plc.ProposalsBucket, plc.CandidatesBucket, plc.PulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, plc.SeatHoldersBucket, - engineReadsBucket, retiredBucket, copiesBucket} + engineReadsBucket, retiredBucket, copiesBucket, writtenBucket} // engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key ``, as // the verb manifest-reads last answered it or the fallback read it: what assigner_engine_reads lists and asks again. @@ -88,6 +88,9 @@ const engineReadsState = "engine-reads" // engineReadsBucket is that state's bucket. const engineReadsBucket = assignment.Module + "_" + engineReadsState +// writtenBucket is the bucket of the state written (pipelines.go): the pipeline ends the wants follow. +const writtenBucket = assignment.Module + "_" + writtenState + // seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants // them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind). var seatBuckets = []string{plc.AssignmentsBucket, plc.ProposalsBucket, plc.CandidatesBucket, @@ -115,6 +118,7 @@ var keptStates = []announce.State{ {Name: engineReadsState, Readable: true}, {Name: retiredState}, // read only by the copy into assignments (migrate.go) {Name: copiesState, Readable: true}, + {Name: writtenState, Readable: true}, } // kvSet is every bucket the assigner opened, by name. @@ -197,7 +201,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any func startsWithout(name string) bool { switch name { case plc.SeatHoldersBucket, buckets.Provisions, assign.PoliciesBucket, assign.FindingsBucket, - buckets.Puts, buckets.Source, buckets.Merges, engineReadsBucket, retiredBucket: + buckets.Puts, buckets.Source, buckets.Merges, engineReadsBucket, retiredBucket, writtenBucket: return true } return slices.Contains(seatBuckets, name) || name == buckets.Grants @@ -297,6 +301,7 @@ var watched = []string{ buckets.Source, buckets.Merges, engineReadsBucket, + writtenBucket, } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after @@ -485,6 +490,14 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, logger.Printf("%s is not heard, so an ask that ends at the operator channel without an answer is not asked again until the assigner starts again: %v", askEndedSubject, err) } readEndedAtBind(ctx, js, hub, kvs[buckets.Asks], staleOfWorld(w), bus, callWait{heard: func() bool { return w.seatHeard(oc.Seat) }, findings: findings}, logger.Printf) + if kv := kvs[writtenBucket]; kv != nil { + // The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow + // once the first registrations are written (§95, §353). + p.heard = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf) + if err := hearEnds(hub, p.heard); err != nil { + logger.Printf("a pipeline's end is not heard, so no want follows a pipeline until the assigner starts again: %v", err) + } + } hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf) } if bus != nil && kvs[buckets.Puts] != nil { diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 477ec9c2..69d369af 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -58,6 +58,10 @@ type passer struct { // neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the // first pass after a start, which clears what a previous process may have raised (startClears). neither map[string]neitherOne + // ends are the pipeline ends the wants follow (pipelines.go); nil is the trunk-only want, kept until the first registrations. + ends *pipelineEnds + // heard is the same follower while the wants do not follow it yet (ends is nil). + heard *pipelineEnds // acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket). acts *putActs // noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is @@ -267,9 +271,16 @@ func (p *passer) wants(ctx context.Context) { moduleGraph := p.moduleGraph() served := p.served(ctx, get, moduleGraph, nodes) offering := p.offering(ctx, get, manifests, nodes) // bus state (ADR 0614 §5) + assignedBranches := p.w.placedBranches() for name := range names { fp := trunk[name].Fp2 name := name + // A want follows a passed pipeline: a module whose fingerprint no pipeline passed at on every branch its placements + // chose, and whose wants do not name it already, gets none (design 47 G §29, §35). + branches := p.ends.branchesOf(name, assignedBranches) + if fp != "" && !p.ends.mayWant(name, fp, branches, wanted[name]) { + continue + } m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp), Self: func(node string) bool { return placed[node] }, Composed: func(node, fp string) (assign.ComposedDigest, bool) { @@ -359,6 +370,9 @@ func (p *passer) wants(ctx context.Context) { want := want p.put(ctx, buckets.Wants, key, func() ([]byte, error) { return mwc.EncodeWant(want) }) } + if len(out.Wants) > 0 { + p.ends.wrote(name, fp, branches) + } } } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go new file mode 100644 index 00000000..743125a5 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -0,0 +1,327 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + mprc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient" + mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" + mpr "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/process" +) + +// The wants follow a passed pipeline (design 47 G §29, §35, §325, §352): the runner says each pipeline's end as +// pipeline-ended, and a group's as group-ended. The assigner keeps, in its old state `written`, one key per module and +// watched branch: the source revision and end count of the newest end it heard for it, and whether the wants were written +// from it. A newer end replaces it by compare-and-set; an older one, or the same one again, writes nothing. + +// writtenState is the assigner's state of the pipeline ends its wants follow; writtenTool reads it. +const ( + writtenState = "written" + writtenTool = "assigner_written" +) + +// pipelineEndedSubject and groupEndedSubject are the runner's events the assigner consumes. +const ( + pipelineEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventPipelineEnded + ".>" + groupEndedSubject = "mesh.seat." + mpr.Seat + ".event." + mpr.EventGroupEnded + ".>" +) + +// End is one value of the state written: the newest end of a module's pipeline on a watched branch. +type End struct { + Source int64 `json:"source"` + EndCount int64 `json:"end-count"` + Fp string `json:"fp"` + Commit string `json:"commit,omitempty"` + // Passed: the pipeline ended passed, every required step passed (and, for a group, the group ended passed). + Passed bool `json:"passed"` + // Written: the wants were written from this end. + Written bool `json:"written,omitempty"` + Why string `json:"why,omitempty"` +} + +// endKey is the key of a module on a watched branch, each name escaped as a key's name is. +func endKey(module, branch string) string { + return keys.EscapeName(module) + "." + keys.EscapeName(branch) +} + +// newer says whether an end of (source, count) replaces the one kept: a later source revision, or the same one ended again. +func (e End) newer(source, count int64) bool { + return source > e.Source || (source == e.Source && count > e.EndCount) +} + +// requiredPassed says whether no required step of a pipeline stands failed, unasked or unended (design 47 G §30: a failed +// optional step holds nothing). +func requiredPassed(p mpr.PipelineEnded) bool { + for _, s := range p.Steps { + if s.Status != mpr.StepResultStatusRequired { + continue + } + switch s.Outcome { + case mpr.StepResultOutcomeFailed, mpr.StepResultOutcomeCouldNotRun, mpr.StepResultOutcomeDeclined, mpr.StepResultOutcomeNotAsked, + mpr.StepResultOutcomeCancelled, mpr.StepResultOutcomeSuperseded, mpr.StepResultOutcomeDidNotBuild, + mpr.StepResultOutcomeRunning, mpr.StepResultOutcomeWaiting, mpr.StepResultOutcomeWaitingForABuildNode: + return false + } + } + return p.Outcome == mpr.PipelineEndedOutcomePassed +} + +// endStore is the kept state written: the bus bucket, or a test's. +type endStore interface { + // get is a key's value and revision; none for a key not there. + get(key string) (value []byte, revision uint64, err error) + // swap writes value at key if the key stands at revision (0: if it does not stand); errSwapped when it does not. + swap(key string, value []byte, revision uint64) error +} + +var errSwapped = errors.New("the key changed") + +// kvEnds is endStore over a bus bucket. +type kvEnds struct { + ctx context.Context + kv jetstream.KeyValue +} + +func (k kvEnds) get(key string) ([]byte, uint64, error) { + e, err := k.kv.Get(k.ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, 0, nil + } + if err != nil { + return nil, 0, err + } + return e.Value(), e.Revision(), nil +} + +func (k kvEnds) swap(key string, value []byte, revision uint64) error { + var err error + if revision == 0 { + _, err = k.kv.Create(k.ctx, key, value) + } else { + _, err = k.kv.Update(k.ctx, key, value, revision) + } + if errors.Is(err, jetstream.ErrKeyExists) || (err != nil && isWrongSequence(err)) { + return errSwapped + } + return err +} + +func isWrongSequence(err error) bool { + var ae *jetstream.APIError + return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence +} + +// pipelineEnds follows the runner's ends. A nil *pipelineEnds is the trunk-only want, kept until the first registrations +// (design 47 G §95, §353). +type pipelineEnds struct { + st endStore + wake func() + logf func(string, ...any) + mu sync.Mutex + // firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design + // 47 G §14). + firsts map[string]string +} + +func newPipelineEnds(st endStore, wake func(), logf func(string, ...any)) *pipelineEnds { + return &pipelineEnds{st: st, wake: wake, logf: logf, firsts: map[string]string{}} +} + +// take keeps one pipeline's end if it is newer than the one kept for its module and branch, by compare-and-set. +func (e *pipelineEnds) take(p mpr.PipelineEnded, passed bool) error { + key := endKey(p.Module, p.Branch) + for { + raw, rev, err := e.st.get(key) + if err != nil { + return err + } + if raw != nil { + var kept End + if err := json.Unmarshal(raw, &kept); err == nil && !kept.newer(p.Source, p.EndCount) { + return nil // an older end, or this one again + } + } + end := End{Source: p.Source, EndCount: p.EndCount, Fp: p.Fp, Commit: p.Commit, Passed: passed, Why: p.Why} + value, err := json.Marshal(end) + if err != nil { + return err + } + if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) { + continue + } else if err != nil { + return err + } + e.wake() + return nil + } +} + +// markWritten says the wants of module on branch were written from the end it keeps, if that is still the end of fp. +func (e *pipelineEnds) markWritten(module, branch, fp string) { + key := endKey(module, branch) + for { + raw, rev, err := e.st.get(key) + var kept End + if err != nil || raw == nil || json.Unmarshal(raw, &kept) != nil || kept.Fp != fp || kept.Written { + return + } + kept.Written = true + value, _ := json.Marshal(kept) + if err := e.st.swap(key, value, rev); errors.Is(err, errSwapped) { + continue + } else if err != nil { + e.logf("the end of %s on %s is not marked written: %v", module, branch, err) + } + return + } +} + +// passedAt says whether the newest end kept for module on branch passed at fp. +func (e *pipelineEnds) passedAt(module, branch, fp string) bool { + raw, _, err := e.st.get(endKey(module, branch)) + var kept End + return err == nil && raw != nil && json.Unmarshal(raw, &kept) == nil && kept.Passed && kept.Fp == fp +} + +// branchOf is the watched branch a placement names, else the first of the module's registration, else the trunk's. +func (e *pipelineEnds) branchOf(module, placed string) string { + if placed != "" { + return placed + } + e.mu.Lock() + defer e.mu.Unlock() + if b := e.firsts[module]; b != "" { + return b + } + return trunkBranch +} + +// hearPipeline takes a pipeline-ended. A pipeline of a group writes nothing on its own end, and a run on request delivers +// nothing (design 47 G §29, §64). +func (e *pipelineEnds) hearPipeline(m jetstream.Msg) { + p, err := mprc.PipelineEnded(m.Data()) + switch { + case err != nil: + e.logf("%s is not taken: %v", m.Subject(), err) + case p.Group != "" || p.Run != "": + default: + if err := e.take(p, requiredPassed(p)); err != nil { + e.logf("%s: %v; it is heard again", m.Subject(), err) + _ = m.Nak() + return + } + } + _ = m.Ack() +} + +// hearGroup takes a group-ended: each member's end, passed only if the group passed. +func (e *pipelineEnds) hearGroup(m jetstream.Msg) { + g, err := mprc.GroupEnded(m.Data()) + if err != nil { + e.logf("%s is not taken: %v", m.Subject(), err) + _ = m.Ack() + return + } + for _, p := range g.Pipelines { + if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil { + e.logf("%s: %v; it is heard again", m.Subject(), err) + _ = m.Nak() + return + } + } + _ = m.Ack() +} + +// hearEnds registers the two events on the assigner's hub. +func hearEnds(hub *process.EventHub, e *pipelineEnds) error { + if err := hub.Handle(pipelineEndedSubject, e.hearPipeline); err != nil { + return fmt.Errorf("%s: %w", pipelineEndedSubject, err) + } + if err := hub.Handle(groupEndedSubject, e.hearGroup); err != nil { + return fmt.Errorf("%s: %w", groupEndedSubject, err) + } + return nil +} + +// writtenTools is the tool assigner_written: every key of the state written, with its end. +func writtenTools(st store) []stdio.Tool { + return []stdio.Tool{{Name: writtenTool, Description: "the pipeline ends the assigner's wants follow: per module and watched branch, the source revision and end count of the newest end heard, whether it passed and whether the wants were written from it.", + Run: func(map[string]any) (any, error) { + ks, err := st.keys(writtenState) + if err != nil { + return nil, err + } + sort.Strings(ks) + out := map[string]End{} + for _, k := range ks { + raw, _, ok, err := st.get(writtenState, k) + if err != nil || !ok { + continue + } + var e End + if json.Unmarshal(raw, &e) == nil { + out[k] = e + } + } + return out, nil + }}} +} + +// branchesOf is the watched branches the placements of module chose, each once; the default branch when none names one. +func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]string) []string { + if e == nil { + return nil + } + seen := map[string]bool{} + var out []string + for _, b := range placed[module] { + b = e.branchOf(module, b) + if !seen[b] { + seen[b] = true + out = append(out, b) + } + } + if len(out) == 0 { + out = append(out, e.branchOf(module, "")) + } + sort.Strings(out) + return out +} + +// mayWant says whether module's wants may name fp: a pipeline passed at fp on every branch, or the wants name fp already +// (a node placed later, a module taken away). +func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[string]mwc.Want) bool { + if e == nil { + return true + } + for _, w := range wanted { + if w.Fp == fp { + return true + } + } + for _, b := range branches { + if !e.passedAt(module, b, fp) { + return false + } + } + return true +} + +// wrote marks the end the wants of module were written from. +func (e *pipelineEnds) wrote(module, fp string, branches []string) { + if e == nil { + return + } + for _, b := range branches { + e.markWritten(module, b, fp) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go new file mode 100644 index 00000000..17597d49 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -0,0 +1,174 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + mpr "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" + + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + + "mesh-delivery-assigner/internal/buckets" +) + +// memEnds is endStore over a map, with revisions. +type memEnds struct { + vals map[string][]byte + revs map[string]uint64 +} + +func newMemEnds() *memEnds { return &memEnds{vals: map[string][]byte{}, revs: map[string]uint64{}} } + +func (m *memEnds) get(key string) ([]byte, uint64, error) { return m.vals[key], m.revs[key], nil } +func (m *memEnds) swap(key string, v []byte, rev uint64) error { + if m.revs[key] != rev { + return errSwapped + } + m.vals[key], m.revs[key] = v, rev+1 + return nil +} + +func pipe(module, branch string, source, count int64, fp, outcome string, steps ...mpr.StepResult) mpr.PipelineEnded { + return mpr.PipelineEnded{Module: module, Branch: branch, Source: source, EndCount: count, Fp: fp, Outcome: outcome, Steps: steps, + Pipeline: branch + "." + module, Commit: "c"} +} + +func step(status, outcome string) mpr.StepResult { + return mpr.StepResult{Step: "unit", Status: status, Outcome: outcome} +} + +// Design 47 G §29, §35, §325, §352 (tests 66, 67, 89): a want follows a passed pipeline, by compare-and-set per module and branch. +func TestAWantFollowsAPassedPipeline(t *testing.T) { + const fp1, fp2 = "fp1", "fp2" + pass, fail := mpr.PipelineEndedOutcomePassed, mpr.PipelineEndedOutcomeFailed + req, opt := mpr.StepResultStatusRequired, mpr.StepResultStatusOptional + for _, c := range []struct { + name string + ends []mpr.PipelineEnded + group *mpr.GroupEnded + module string + branch string + fp string + may bool + }{ + {"a failed required step writes no want", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed"))}, nil, "web", "main", fp1, false}, + {"a pipeline passed writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true}, + {"a failed optional step holds nothing", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass, step(req, "passed"), step(opt, "failed"))}, nil, "web", "main", fp1, true}, + {"a pipeline passed at another fingerprint", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp2, false}, + {"an older revision ending after a newer one writes nothing", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "main", 5, 1, fp1, pass)}, nil, "web", "main", fp1, false}, + {"another branch writes its own", []mpr.PipelineEnded{pipe("web", "main", 6, 1, fp2, fail), pipe("web", "beta", 5, 1, fp1, pass)}, nil, "web", "beta", fp1, true}, + {"a later end of the same revision writes", []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, fail, step(req, "failed")), pipe("web", "main", 5, 2, fp1, pass, step(req, "passed"))}, nil, "web", "main", fp1, true}, + {"the same end again after a later one keeps the later", []mpr.PipelineEnded{pipe("web", "main", 5, 2, fp1, pass), pipe("web", "main", 5, 1, fp1, fail)}, nil, "web", "main", fp1, true}, + {"a member's own end writes nothing", []mpr.PipelineEnded{func() mpr.PipelineEnded { p := pipe("web", "main", 5, 1, fp1, pass); p.Group = "g"; return p }()}, nil, "web", "main", fp1, false}, + {"a group ended passed writes its members", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomePassed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, true}, + {"a group ended failed writes none", nil, &mpr.GroupEnded{Branch: "main", Group: "g", Outcome: mpr.GroupEndedOutcomeFailed, Pipelines: []mpr.PipelineEnded{pipe("web", "main", 5, 1, fp1, pass)}}, "web", "main", fp1, false}, + } { + woken := 0 + e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf) + for _, p := range c.ends { + if p.Group != "" { + continue // heard and ignored, as hearPipeline does + } + if err := e.take(p, requiredPassed(p)); err != nil { + t.Fatal(err) + } + } + if g := c.group; g != nil { + for _, p := range g.Pipelines { + if err := e.take(p, g.Outcome == mpr.GroupEndedOutcomePassed && requiredPassed(p)); err != nil { + t.Fatal(err) + } + } + } + if got := e.mayWant(c.module, c.fp, []string{c.branch}, nil); got != c.may { + t.Errorf("%s: may want = %v, want %v", c.name, got, c.may) + } + } + // A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it). + e := newPipelineEnds(newMemEnds(), func() {}, t.Logf) + if !e.mayWant("web", fp1, []string{"main"}, map[string]mwc.Want{"g14": {Fp: fp1}}) { + t.Error("a want naming the fingerprint was held") + } + // markWritten keeps the end and says it was written from; a newer end replaces it unwritten. + st := newMemEnds() + e = newPipelineEnds(st, func() {}, t.Logf) + _ = e.take(pipe("web", "main", 5, 1, fp1, pass), true) + e.wrote("web", fp1, []string{"main"}) + var end End + _ = json.Unmarshal(st.vals[endKey("web", "main")], &end) + if !end.Written || end.Source != 5 { + t.Errorf("end = %+v, want written at source 5", end) + } + _ = e.take(pipe("web", "main", 6, 1, fp2, pass), true) + end = End{} + _ = json.Unmarshal(st.vals[endKey("web", "main")], &end) + if end.Written || end.Source != 6 { + t.Errorf("a newer end = %+v, want unwritten at source 6", end) + } +} + +// Design 47 G §29: the wants of a pass follow a pipeline: with the follower on, a module whose fingerprint no pipeline passed at +// gets no want and one a pipeline passed at gets it; with it off (nil, the trunk-only want) the pass is as before. +func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + for _, c := range []struct { + name string + follow bool + passed bool + want bool + }{{"the trunk-only want", false, false, true}, {"no pipeline passed", true, false, false}, {"a pipeline passed", true, true, true}} { + wantsKV := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Wants: wantsKV, buckets.Asks: &memKV{values: map[string][]byte{}}}, time.Now()) + p.say = func(oc.Condition) error { return nil } + mdigest := strings.Repeat("e", 64) + mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: mdigest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath}) + p.w.put(buckets.Modules, "web", mod) + p.w.manifests[mdigest] = parsed{module: "web", manifest: catalogue.Manifest{Module: "web"}} + bkey, _ := builds.BuilderBuildKey("web", fp) + built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: mdigest}) + p.w.put(buckets.Builds, bkey, built) + gkey, _ := grantsstate.PartGrantKey("web", fp) + grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{}) + p.w.put(buckets.Grants, gkey, grant) + digest := "sha256:" + strings.Repeat("c", 64) + have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: digest, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) + p.w.put(buckets.Have, "g14.web", have) + composes(t, p.w, "g14", digest) + placeOn(t, p.w, "g14", "web") + on, _ := presenceValue(true) + p.w.put(buckets.Presence, "g14", on) + if c.follow { + p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + if c.passed { + _ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true) + } + } + p.wants(context.Background()) + key, _ := mwc.WantKey("g14", "web") + if _, got := wantsKV.values[key]; got != c.want { + t.Errorf("%s: a want written = %v, want %v", c.name, got, c.want) + } + if c.follow && c.passed { + raw, _, _ := p.ends.st.get(endKey("web", trunkBranch)) + var end End + _ = json.Unmarshal(raw, &end) + if !end.Written { + t.Errorf("%s: the end was not marked written", c.name) + } + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index d81e9acf..9b946428 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -404,6 +404,7 @@ func serveTools() error { tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...) // What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it. tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now)) + tools = append(tools, writtenTools(runtimeStore{})...) tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index 893d02f5..a5a1ca9a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -578,3 +578,17 @@ func (w *world) unplacedNodes() []string { // trunkBranch is the branch of the trunk-only want, kept until the first registrations (design 47 G §95, §353). const trunkBranch = "main" + +// placedBranches is, by module and node, the watched branch each placement names ("" when it names none). +func (w *world) placedBranches() map[string]map[string]string { + out := map[string]map[string]string{} + for node, a := range w.assignments() { + for _, m := range a.Modules { + if out[m.Module] == nil { + out[m.Module] = map[string]string{} + } + out[m.Module][node] = m.Branch + } + } + return out +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index c1ceeaea..629dcee0 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -65,7 +65,7 @@ func TestManifest(t *testing.T) { } if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20, "assignments": 64, "placement": 64, "copies": 1, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, views.MarksState: 1, views.StatusState: 1, - "engine-reads": 1}; !reflect.DeepEqual(states, want) { + "engine-reads": 1, "written": 5}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -96,7 +96,7 @@ func TestManifest(t *testing.T) { // which lists and asks again what each node's engine reads. if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assignments", "seat-holders", "wants", "assigner_read_assignments", "assigner_read_seat_holders", "assigner_read_wants", "assigner_read_findings", "assigner_read_asks", "assigner_read_differences", "assigner_read_copies", - "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again"}) { + "assigner_first_views", "assigner_first_view_again", "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", "assigner_ask_again", "assigner_written"}) { t.Errorf("the assigner serves tools %v, want its three put tools, three first views, six read tools, assigner_engine_reads, the seat list's two (ADR 0636 §7) and assigner_ask_again (ADR 0668 §5)", m.Tools) } if m.RunsAs != m.Module || m.SecretsOwner != m.RunsAs { @@ -123,7 +123,7 @@ func TestManifest(t *testing.T) { } // ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events; // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears. - if !slices.Equal(m.Consumes, []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "node-engine.manifest-reads-changed"}) { + if !slices.Equal(m.Consumes, []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) { t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes) } if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index e594b25c..d81cca86 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -45,6 +45,8 @@ "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", "mesh-delivery-teller.ask-ended", + "mesh-pipeline-runner.pipeline-ended", + "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed" ], "invokes": [ @@ -90,7 +92,8 @@ "assigner_engine_reads", "assigner_seat_list", "assigner_seat_list_again", - "assigner_ask_again" + "assigner_ask_again", + "assigner_written" ], "state": [ { "name": "assignments", "history": 64 }, @@ -108,7 +111,8 @@ { "name": "seat-holders", "history": 5 }, { "name": "view-marks", "history": 1 }, { "name": "first-views", "history": 1 }, - { "name": "engine-reads", "history": 1 } + { "name": "engine-reads", "history": 1 }, + { "name": "written", "history": 5 } ], "own-secrets": { "broker": "${dir:state}/broker" From 61277b6b9b47e651f8e7b19fcfbf2f9574d2ca4b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:53:41 +0100 Subject: [PATCH 12/16] VW4: the view's manifest says version 0.1, declares its finding stream and hears the keeper's ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §383, §387, §368, §411 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-view/manifest_test.go | 17 ++++++++++++++++- modules/mesh-delivery-view/module.json | 11 ++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index a2e58671..b35cf6f8 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -51,6 +51,13 @@ type state struct { History int `json:"history,omitempty"` } +type stream struct { + Name string `json:"name"` + Events []string `json:"events"` + MaxAgeDays int `json:"max-age-days"` + MaxMiB int `json:"max-mib"` +} + type manifest struct { State []state `json:"state"` Module string `json:"module"` @@ -63,6 +70,7 @@ type manifest struct { Uses []string `json:"uses"` Tools []string `json:"tools"` Emits []string `json:"emits"` + Streams []stream `json:"streams"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` Resources []map[string]any `json:"resources"` @@ -117,7 +125,14 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 "seat:node-engine.manifest-reads"}; !reflect.DeepEqual(m.Invokes, want) { // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node; 1585/T24: view_tiers asks indexer_pending, a verb edge (ADR 0614 §2) t.Errorf("invokes = %v, want %v", m.Invokes, want) } - if want := []string{"mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", + // design 47 G §387: the keeper's ask for the findings is heard, the finding stream declared (§383), and §368: version is MAJOR.MINOR. + if want := []stream{{Name: "mesh-delivery-view-findings", Events: []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}, MaxAgeDays: 7, MaxMiB: 16}}; !reflect.DeepEqual(m.Streams, want) { + t.Errorf("streams = %+v, want %+v", m.Streams, want) + } + if m.Version != "0.1" { + t.Errorf("version = %q, want 0.1", m.Version) + } + if want := []string{"mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.candidates", "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken"}; !reflect.DeepEqual(m.Consumes, want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 81d90b74..a8544e42 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -1,6 +1,6 @@ { "module": "mesh-delivery-view", - "version": "1", + "version": "0.1", "moved": true, "slug": "view", "tools": [ @@ -42,6 +42,7 @@ { "name": "findings", "history": 1 } ], "consumes": [ + "mesh-conditions.findings-asked", "mesh-delivery-assigner.views.asks", "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", @@ -49,6 +50,14 @@ "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.taken" ], + "streams": [ + { + "name": "mesh-delivery-view-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "invokes": [ "mesh-delivery-assigner.assigner_read_asks", "mesh-delivery-assigner.assigner_read_differences", From 1778888acca14459a3ccc23ecceda029921cd578 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:54:50 +0100 Subject: [PATCH 13/16] AS2: the wants keep the trunk-only form until the first registration, and follow the pipelines' ends from it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §95, §130, §353, §352 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/main.go | 10 +- .../cmd/mesh-delivery-assigner/pass.go | 4 +- .../cmd/mesh-delivery-assigner/pipelines.go | 96 ++++++++++++++++++- .../mesh-delivery-assigner/pipelines_test.go | 39 ++++++++ .../mesh-delivery-assigner/manifest_test.go | 4 +- modules/mesh-delivery-assigner/module.json | 2 + 6 files changed, 143 insertions(+), 12 deletions(-) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index 4f040bed..3ed9dab2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -493,9 +493,13 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if kv := kvs[writtenBucket]; kv != nil { // The runner's pipeline-ended and group-ended (design 47 G §29): kept in the state written, which the wants follow // once the first registrations are written (§95, §353). - p.heard = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf) - if err := hearEnds(hub, p.heard); err != nil { - logger.Printf("a pipeline's end is not heard, so no want follows a pipeline until the assigner starts again: %v", err) + p.ends = newPipelineEnds(kvEnds{ctx, kv}, wake, logger.Printf) + err := hearEnds(hub, p.ends) + if err == nil && bus != nil { + err = followRegistrations(ctx, hub, bus.Call, p.ends) + } + if err != nil { + logger.Printf("a pipeline's end or a registration is not heard, so the wants keep the trunk-only form until the assigner starts again: %v", err) } } hearReadsChanged(ctx, js, hub, reads, findings, gate.eventsLive, logger.Printf) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index 69d369af..8c5d6aa2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -58,10 +58,8 @@ type passer struct { // neither is, by node and module, each module on neither list whose condition this process raised (issue 1945). Nil until the // first pass after a start, which clears what a previous process may have raised (startClears). neither map[string]neitherOne - // ends are the pipeline ends the wants follow (pipelines.go); nil is the trunk-only want, kept until the first registrations. + // ends are the pipeline ends the wants follow (pipelines.go), once the first registrations are heard; until then, and nil, the trunk-only want. ends *pipelineEnds - // heard is the same follower while the wants do not follow it yet (ends is nil). - heard *pipelineEnds // acts are the puts' acts (ADR 0546); nil: the puts are not taken further (no bus to call, or no puts bucket). acts *putActs // noWant is, by node and module, the reason last said why the node gets no want for the module (issue 1945): a reason is diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go index 743125a5..5e9ab67e 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines.go @@ -5,13 +5,19 @@ import ( "encoding/json" "errors" "fmt" + "slices" "sort" + "strings" "sync" + "sync/atomic" + "time" "github.com/nats-io/nats.go/jetstream" + mcc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" mprc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshpipelinerunnerclient" mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" + mpcat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" mpr "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/keys" @@ -81,6 +87,9 @@ type endStore interface { get(key string) (value []byte, revision uint64, err error) // swap writes value at key if the key stands at revision (0: if it does not stand); errSwapped when it does not. swap(key string, value []byte, revision uint64) error + // keys are the keys that stand, and drop deletes one. + keys() ([]string, error) + drop(key string) error } var errSwapped = errors.New("the key changed") @@ -115,6 +124,16 @@ func (k kvEnds) swap(key string, value []byte, revision uint64) error { return err } +func (k kvEnds) keys() ([]string, error) { + ks, err := k.kv.Keys(k.ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + return ks, err +} + +func (k kvEnds) drop(key string) error { return k.kv.Delete(k.ctx, key) } + func isWrongSequence(err error) bool { var ae *jetstream.APIError return errors.As(err, &ae) && ae.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence @@ -126,7 +145,9 @@ type pipelineEnds struct { st endStore wake func() logf func(string, ...any) - mu sync.Mutex + // on is whether the wants follow the ends: set by the first registration heard, never before (design 47 G §95, §353). + on atomic.Bool + mu sync.Mutex // firsts is, by module, the first watched branch of its registration, the branch of a placement that names none (design // 47 G §14). firsts map[string]string @@ -278,7 +299,7 @@ func writtenTools(st store) []stdio.Tool { // branchesOf is the watched branches the placements of module chose, each once; the default branch when none names one. func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]string) []string { - if e == nil { + if !e.following() { return nil } seen := map[string]bool{} @@ -300,7 +321,7 @@ func (e *pipelineEnds) branchesOf(module string, placed map[string]map[string]st // mayWant says whether module's wants may name fp: a pipeline passed at fp on every branch, or the wants name fp already // (a node placed later, a module taken away). func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[string]mwc.Want) bool { - if e == nil { + if !e.following() { return true } for _, w := range wanted { @@ -318,10 +339,77 @@ func (e *pipelineEnds) mayWant(module, fp string, branches []string, wanted map[ // wrote marks the end the wants of module were written from. func (e *pipelineEnds) wrote(module, fp string, branches []string) { - if e == nil { + if !e.following() { return } for _, b := range branches { e.markWritten(module, b, fp) } } + +// following says whether the wants follow the pipeline ends yet; a nil follower never does. +func (e *pipelineEnds) following() bool { return e != nil && e.on.Load() } + +// registered takes a module's registration (design 47 G §353): the first one heard switches the wants to the ends, its first +// watched branch is the default a placement runs, and the ends kept for a branch it no longer watches are deleted (§352). gone +// is a registration removed: all its ends go. +func (e *pipelineEnds) registered(reg mpcat.Registration, gone bool) { + e.mu.Lock() + if gone || len(reg.Branches) == 0 { + delete(e.firsts, reg.Module) + } else { + e.firsts[reg.Module] = reg.Branches[0] + } + e.mu.Unlock() + kept, err := e.st.keys() + if err != nil { + e.logf("the ends kept cannot be listed: %v", err) + } + prefix := keys.EscapeName(reg.Module) + "." + for _, k := range kept { + rest, ok := strings.CutPrefix(k, prefix) + if !ok { + continue + } + branch, err := keys.UnescapeName(rest) + if err == nil && !gone && slices.Contains(reg.Branches, branch) { + continue + } + if err := e.st.drop(k); err != nil { + e.logf("the end %s is not deleted: %v", k, err) + } + } + if !e.on.Swap(true) { + e.logf("the first registration is heard (%s): the wants follow the pipelines' ends from now", reg.Module) + } + e.wake() +} + +// followRegistrations follows the registrations through the seat mesh-catalog (design 47 G §353): a first view of its verb +// and each registration-changed on the hub. It returns when the hub's handler is registered; the first view is asked on its own +// goroutine, the wants keep the trunk-only form until a registration is heard. +func followRegistrations(ctx context.Context, hub *process.EventHub, call mcc.Call, e *pipelineEnds) error { + regs := mcc.NewRegistrations(call) + regs.Follower.Logf = e.logf + regs.Follower.Changed = func(module string, value json.RawMessage, gone bool, _ time.Time) { + reg := mpcat.Registration{Module: module} + if !gone { + _ = json.Unmarshal(value, ®) + } + e.registered(reg, gone) + } + if err := hub.Handle(mpcat.SubjectRegistrationChanged, func(m jetstream.Msg) { + if err := regs.Hear(ctx, m.Data()); err != nil { + e.logf("%s: %v", m.Subject(), err) + } + _ = m.Ack() + }); err != nil { + return err + } + go func() { + if err := regs.Start(ctx); err != nil && ctx.Err() == nil { + e.logf("the first view of the registrations is not answered: %v", err) + } + }() + return nil +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go index 17597d49..acb0c207 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pipelines_test.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" mwc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + mpcat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" mpr "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshpipelinerunner" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" @@ -34,6 +35,13 @@ type memEnds struct { func newMemEnds() *memEnds { return &memEnds{vals: map[string][]byte{}, revs: map[string]uint64{}} } func (m *memEnds) get(key string) ([]byte, uint64, error) { return m.vals[key], m.revs[key], nil } +func (m *memEnds) keys() (out []string, _ error) { + for k := range m.vals { + out = append(out, k) + } + return out, nil +} +func (m *memEnds) drop(key string) error { delete(m.vals, key); delete(m.revs, key); return nil } func (m *memEnds) swap(key string, v []byte, rev uint64) error { if m.revs[key] != rev { return errSwapped @@ -79,6 +87,7 @@ func TestAWantFollowsAPassedPipeline(t *testing.T) { } { woken := 0 e := newPipelineEnds(newMemEnds(), func() { woken++ }, t.Logf) + e.on.Store(true) for _, p := range c.ends { if p.Group != "" { continue // heard and ignored, as hearPipeline does @@ -100,12 +109,14 @@ func TestAWantFollowsAPassedPipeline(t *testing.T) { } // A want that names the fingerprint already is not held by a later failure (a node placed afterwards gets it). e := newPipelineEnds(newMemEnds(), func() {}, t.Logf) + e.on.Store(true) if !e.mayWant("web", fp1, []string{"main"}, map[string]mwc.Want{"g14": {Fp: fp1}}) { t.Error("a want naming the fingerprint was held") } // markWritten keeps the end and says it was written from; a newer end replaces it unwritten. st := newMemEnds() e = newPipelineEnds(st, func() {}, t.Logf) + e.on.Store(true) _ = e.take(pipe("web", "main", 5, 1, fp1, pass), true) e.wrote("web", fp1, []string{"main"}) var end End @@ -153,6 +164,7 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { p.w.put(buckets.Presence, "g14", on) if c.follow { p.ends = newPipelineEnds(newMemEnds(), func() {}, t.Logf) + p.ends.on.Store(true) if c.passed { _ = p.ends.take(pipe("web", trunkBranch, 5, 1, fp, mpr.PipelineEndedOutcomePassed), true) } @@ -172,3 +184,30 @@ func TestAPassWritesNoWantWithoutAPassedPipeline(t *testing.T) { } } } + +// Design 47 G §95, §130, §353, §352 (test 100): the wants keep the trunk-only form until the first registration is heard and +// switch on it; a branch a registration no longer watches loses its end, and its first branch is a placement's default. +func TestTheWantsSwitchOnTheFirstRegistration(t *testing.T) { + st := newMemEnds() + woken := 0 + e := newPipelineEnds(st, func() { woken++ }, t.Logf) + _ = e.take(pipe("web", "main", 5, 1, "fp1", mpr.PipelineEndedOutcomeFailed), false) + _ = e.take(pipe("web", "beta", 5, 1, "fp1", mpr.PipelineEndedOutcomePassed), true) + if e.following() || !e.mayWant("web", "fp1", []string{"main"}, nil) { + t.Fatal("the follower held a want before any registration") + } + e.registered(mpcat.Registration{Module: "web", Branches: []string{"beta"}}, false) + if !e.following() || woken == 0 { + t.Error("the first registration did not switch the wants on") + } + if e.mayWant("web", "fp1", []string{"main"}, nil) || !e.mayWant("web", "fp1", []string{"beta"}, nil) { + t.Error("after the switch the wants did not follow the ends") + } + if len(st.vals) != 1 || e.branchOf("web", "") != "beta" || e.branchOf("other", "") != trunkBranch { + t.Errorf("ends kept %d, default branch %q: an unwatched branch's end stayed, or the default is wrong", len(st.vals), e.branchOf("web", "")) + } + e.registered(mpcat.Registration{Module: "web"}, true) + if len(st.vals) != 0 { + t.Errorf("a registration gone left %d ends", len(st.vals)) + } +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 629dcee0..036dbc34 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -123,11 +123,11 @@ func TestManifest(t *testing.T) { } // ADR 0618 §4: and it hears each node's engine say its reads changed; ADR 0636 §4 (issue 1966): and the seat list's two events; // issue 1966 note 151: and the teller's ask-ended, its subject the one the assigner hears. - if !slices.Equal(m.Consumes, []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) { + if !slices.Equal(m.Consumes, []string{"mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", "mesh-delivery-indexer.views.manifests", "mesh-delivery-indexer.views.modules", buckets.TellerModule + "." + tellerevents.AskEnded, "mesh-pipeline-runner.pipeline-ended", "mesh-pipeline-runner.group-ended", "node-engine.manifest-reads-changed"}) { t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes) } if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", - "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list + "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-engine.manifest-reads", // ADR 0618 §3 diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index d81cca86..d6252329 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -39,6 +39,7 @@ "operator-channel" ], "consumes": [ + "mesh-catalog.registration-changed", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-indexer.views.graph", @@ -53,6 +54,7 @@ "mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", + "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", From 0b597eb41749cbdbe576536e28bb431aaa4049ee Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:54:56 +0100 Subject: [PATCH 14/16] FB3, FB4, FB5: the bridge performs what the gate owes through the seat git and says each outcome, reads groups from the seat, and its owed tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Taken over from the join (6ada2089, f1417ab0, 6e162a60, 3a4e62b0, 73fb42d7, ee15a280, a5476f62, 94a4e62a, 33907a81, e519cc87, e8f268c1) and fitted to the final contracts: the seat's events are pull-updated, pull-merged, pull-closed, forge-delete; OwedEnded keeps only the fields §164 names; merge-pull takes no ask. A merge act is performed only as a switched act (§169); from G the gate owes none (§167). Builds: design 47 G §43, §45, §157, §161, §162, §163, §164, §167, §168, §169, §171, §250 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../clients/meshdeliveryclient/gate_test.go | 31 + .../mesh-delivery-forge-bridge-tools/main.go | 209 +++--- .../main_test.go | 148 ++-- .../mesh-delivery-forge-bridge-tools/owed.go | 132 ++++ .../cmd/mesh-delivery-forge-bridge/gate.go | 132 ---- .../mesh-delivery-forge-bridge/gate_test.go | 381 +++++----- .../mesh-delivery-forge-bridge/groupask.go | 44 +- .../cmd/mesh-delivery-forge-bridge/groups.go | 162 ++--- .../mesh-delivery-forge-bridge/groups_test.go | 109 ++- .../integration_test.go | 688 +++++++++--------- .../cmd/mesh-delivery-forge-bridge/main.go | 82 ++- .../cmd/mesh-delivery-forge-bridge/names.go | 8 +- .../mesh-delivery-forge-bridge/names_test.go | 20 +- .../cmd/mesh-delivery-forge-bridge/owed.go | 342 +++++++++ .../forgemsg/forge.go | 2 +- modules/mesh-delivery-forge-bridge/go.mod | 2 + .../internal/bridge/acts.go | 160 ---- .../internal/bridge/acts_test.go | 188 ----- .../internal/bridge/call.go | 2 +- .../internal/bridge/owed.go | 147 ++++ .../internal/bridge/owed_test.go | 95 +++ .../internal/gitfixture/gitfixture.go | 9 +- .../internal/groups/groups.go | 157 ++-- .../internal/groups/owedask.go | 95 +++ .../manifest_test.go | 48 +- .../mesh-delivery-forge-bridge/module.json | 39 +- 26 files changed, 1940 insertions(+), 1492 deletions(-) create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go create mode 100644 modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go delete mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/acts.go delete mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/owed.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go create mode 100644 modules/mesh-delivery-forge-bridge/internal/groups/owedask.go diff --git a/contracts/go/clients/meshdeliveryclient/gate_test.go b/contracts/go/clients/meshdeliveryclient/gate_test.go index 92be0bf2..afcee870 100644 --- a/contracts/go/clients/meshdeliveryclient/gate_test.go +++ b/contracts/go/clients/meshdeliveryclient/gate_test.go @@ -142,6 +142,37 @@ func TestGateKeysAsGoDeliveryHadThem(t *testing.T) { t.Errorf("%q reads as a switched act's id", other) } } + // A switched act's id carries its state, key and revision, and reads back; any other id is not one. + for _, c := range []struct { + state, key string + rev uint64 + id string + }{ + {SwitchedOwed, "novox.app.7.0123abcd.status=mesh/delivery", 4, "switched.owed.4.novox.app.7.0123abcd.status=mesh/delivery"}, + {SwitchedMerges, "release-g", 12, "switched.merges.12.release-g"}, + } { + id, err := SwitchedActID(c.state, c.key, c.rev) + if err != nil || id != c.id { + t.Errorf("SwitchedActID(%q, %q, %d) = %q, %v; want %q", c.state, c.key, c.rev, id, err, c.id) + } + if s, k, r, ok := ParseSwitchedActID(id); !ok || s != c.state || k != c.key || r != c.rev { + t.Errorf("ParseSwitchedActID(%q) = %q %q %d %v", id, s, k, r, ok) + } + } + for _, bad := range []struct { + state, key string + rev uint64 + }{{"turns", "k", 1}, {SwitchedOwed, "", 1}, {SwitchedOwed, "a@b", 1}, {SwitchedMerges, "k", 0}} { + if id, err := SwitchedActID(bad.state, bad.key, bad.rev); err == nil { + t.Errorf("SwitchedActID(%+v) = %q, no error", bad, id) + } + } + for _, other := range []string{"novox.app.7.0123abcd.view@1700000000", "switched.turns.1.k", "switched.owed.x.k", + "switched.owed.0.k", "switched.owed.3", "switched.owed.3.k@1"} { + if _, _, _, ok := ParseSwitchedActID(other); ok { + t.Errorf("%q reads as a switched act's id", other) + } + } } func TestGateValuesRefuseWhatTheirRulesRefuse(t *testing.T) { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go index d123a475..4bc1619d 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main.go @@ -9,17 +9,13 @@ import ( "context" "encoding/json" "fmt" - "maps" "os" - "slices" - "strconv" "strings" "sync" "time" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/sdk/trunk" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" @@ -28,9 +24,6 @@ import ( "mesh-delivery-forge-bridge/internal/groups" ) -// gateMergesState is the gate's state merges, which the tools read (reads: mesh-delivery-gate.merges in the manifest). -const gateMergesState = "mesh-delivery-gate.merges" - // Tool names. const ( GroupsTool = "bridge_groups" @@ -42,8 +35,8 @@ const ( // asker writes an end-asks value and waits for the bridge's answer to it. type asker func(key string, value []byte) (groups.EndAsk, error) -// Tools are the bridge's tools: groupsRead reads the groups the gate holds with what the bridge kept of them. -func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string, number int) (json.RawMessage, bool, error), forge bridge.Caller) []stdio.Tool { +// Tools are the bridge's tools: groupsRead reads the seat mesh-delivery's groups with what the bridge kept of them. +func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []string, error), merged func(repository, merge string, number int) (json.RawMessage, error), forge bridge.Caller) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } return []stdio.Tool{ {Name: bridge.TagsTool, @@ -59,37 +52,32 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s {Name: MergedTool, Description: "The merged message of one merge, asked again (issue 1585): the message the bridge wrote on its stream FORGE when the " + "pull request was merged into the trunk, built the same way from the forge (the merge commit's trees and the manifests " + - "and lock files of its module directories) and from the gate's merges state (the pull request's number, head and " + - "group). For the indexer, which asks it when it takes in a held merge it does not hold in memory (a restart, a new " + + "and lock files of its module directories), the pull request's number, branch and head from the seat git's verb pull, and " + + "for a delivery group's member its group and members from the seat mesh-delivery's verb groups. For the indexer, which asks it when it takes in a held merge it does not hold in memory (a restart, a new " + "build), and the ask-again a lost message lacked. About 0.5 MB for a repository of many modules. Answers {message} " + - "with the forge message whole, or why it cannot: neither the gate nor the forge gives a merged pull request with that commit, or the forge did not answer. " + - "A merge made on the forge (a recovery merge, the operator's own) has no merge in the gate: the bridge reads that pull request from the forge, by number when given, and says made_on_forge.", + "with the forge message whole, or why it cannot: the forge gives no merged pull request with that commit, or a seat did not answer. " + + "A merge made on the forge (a recovery merge, the operator's own) is read the same way.", Input: map[string]any{"type": "object", "required": []string{"repository", "merge"}, "properties": map[string]any{ "repository": map[string]any{"type": "string", "description": "the repository, owner/name, as the forge names it"}, "merge": map[string]any{"type": "string", "description": "the merge commit, 40 hex digits"}, - "number": map[string]any{"type": "integer", "description": "the pull request's number, when known: for a merge made on the forge the bridge reads that pull request and checks its merge commit"}}}, + "number": map[string]any{"type": "integer", "description": "the pull request's number, when known: the bridge reads that pull request and checks its merge commit"}}}, Run: func(args map[string]any) (any, error) { number, _ := args["number"].(float64) - raw, onForge, err := merged(str(args, "repository"), str(args, "merge"), int(number)) + raw, err := merged(str(args, "repository"), str(args, "merge"), int(number)) if err != nil { return nil, err } - out := map[string]any{"message": raw} - if onForge { - out["made_on_forge"] = true - out["note"] = "the merge was made on the forge, not through the gate: its pull request was read from the forge" - fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] merged message of %s at %s answered from the forge: the merge was made on the forge, not through the gate\n", str(args, "repository"), str(args, "merge")) - } - return out, nil + return map[string]any{"message": raw}, nil }}, {Name: GroupsTool, - Description: "The delivery groups the gate holds (more than one member), each with its members' merge state and whether each " + - "merged message is written on the forge stream, and whether the group's end (forge.group.merged) is written: ended, " + - "or open with why (gathered, or merged and its end not written yet). An open group with every member merged is " + + Description: "The delivery groups the seat mesh-delivery's verb groups answers (more than one member), each with its members' delivery " + + "state and whether each merged message is written on the forge stream (with the merge commit it named), and whether the " + + "group's end (forge.group.merged) is written: ended, or open with why (gathered, or every message written and its end " + + "not written yet). An open group with every member's message written is " + "what bridge_group_end_again says the end of. Each group names its last end ask (who, why, when) and the bridge's " + "answer to it, from the state end-asks. A group's name narrows it to that group; open lists only open ones.", Input: map[string]any{"type": "object", "properties": map[string]any{ - "group": map[string]any{"type": "string", "description": "only this group, by the name the gate keys it"}, + "group": map[string]any{"type": "string", "description": "only this group, by its id (its branch)"}, "open": map[string]any{"type": "boolean", "description": "only the groups whose end is not written"}}}, Run: func(args map[string]any) (any, error) { rows, unread, err := groupsRead() @@ -110,11 +98,11 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s {Name: EndAgainTool, Description: "Say a delivery group's end (forge.group.merged) again, by hand, for a group whose end the indexer did not take " + "in (a group bridge_groups names as merged with its end not written, or one the indexer waits for). The bridge " + - "writes it again from the gate's merges, with the time appended to the message id so the stream does not drop it as " + + "writes it again from the seat mesh-delivery's groups, with the time appended to the message id so the stream does not drop it as " + "a duplicate; the indexer takes an end for a group it already took in without changing anything. Refused while " + - "a member is not merged or its merged message is not written. Answers whether it was written.", + "a member's merged message is not written. Answers whether it was written.", Input: map[string]any{"type": "object", "required": []string{"group"}, "properties": map[string]any{ - "group": map[string]any{"type": "string", "description": "the group, by the name the gate keys it"}, + "group": map[string]any{"type": "string", "description": "the group, by its id (its branch)"}, "why": map[string]any{"type": "string", "description": "why its end is said again, in plain words"}}}, Run: func(args map[string]any) (any, error) { key, value, err := groups.NewEndAsk(str(args, "group"), str(args, "why"), stdio.CallerOf(args), now()) @@ -131,87 +119,50 @@ func Tools(ask asker, now func() time.Time, groupsRead func() ([]groups.Row, []s } // MergedOf is the merged message of the merge of repository at commit merge, as the bridge writes it: the pull request's -// number, head and group from the gate's merges (raw, by key), the rest from the forge through trees and files -// (bridge.MergedMessage), and, for a member of a delivery group, the group and its members as the process sets them. -// -// When the gate holds no merge with that commit, the merge was made on the forge (a recovery merge, the operator's own): -// the pull request is read through the seat git's verb pull (by number when given, else by the merge commit), the message is -// built the same way, and madeOnForge is true, for the answer and its log line to say. It is a lone pull request: no -// group is set. -func MergedOf(raws map[string][]byte, repository, merge string, number int, trees bridge.Trees, files bridge.Files, forge bridge.Caller) (out json.RawMessage, madeOnForge bool, err error) { +// number, branch and head through the seat git's verb pull (by number when given, else by the merge commit; a merge made +// on the forge is read the same way), the rest from the forge through trees and files (bridge.MergedMessage), and, for a +// member of a delivery group the seat mesh-delivery names (of), the group and its members as the process sets them. +func MergedOf(of []seat.Group, repository, merge string, number int, trees bridge.Trees, files bridge.Files, forge bridge.Caller) (json.RawMessage, error) { owner, repo, ok := strings.Cut(repository, "/") if !ok || owner == "" || repo == "" { - return nil, false, fmt.Errorf("%q is not owner/name", repository) - } - for _, key := range slices.Sorted(maps.Keys(raws)) { - m, err := meshdeliveryclient.DecodeMerge(raws[key]) - if err != nil { - continue - } - at := slices.IndexFunc(m.Pulls, func(p seat.MergesByGroupValuePullsItem) bool { - return trunk.SameRepository(p.Repository, repository) && p.Commit == merge - }) - if at < 0 { - continue - } - p := m.Pulls[at] - msg, err := bridge.MergedMessage(owner, repo, int(p.Number), m.Group, p.Head, merge, trees, files) - if err != nil { - return nil, false, fmt.Errorf("the merged message of %s at %s cannot be built: %w", repository, merge, err) - } - f, err := forgemsg.DecodeForge(msg.Data) - if err != nil { - return nil, false, err - } - if _, approved := m.Approved[meshdeliveryclient.ApprovedKey(f.Repository, f.Number)]; approved && len(m.Approved) >= 2 { - f.Group, f.Members = m.Group, groups.MembersOf(m) - } - out, err = forgemsg.EncodeForge(f) - return out, false, err - } - if forge == nil { - return nil, false, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, so its pull request is not known", repository, merge) + return nil, fmt.Errorf("%q is not owner/name", repository) } fp, err := bridge.ForgeMade(context.Background(), forge, owner, repo, number, merge) if err != nil { - return nil, false, fmt.Errorf("the gate's merges hold no merge of %s with the commit %s, and the forge does not give it as a merged pull request either: %w", repository, merge, err) + return nil, fmt.Errorf("the forge gives no merged pull request of %s with the commit %s: %w", repository, merge, err) } msg, err := bridge.MergedMessage(owner, repo, fp.Number, fp.Branch, fp.Head, merge, trees, files) if err != nil { - return nil, false, fmt.Errorf("the merged message of %s at %s (a merge made on the forge) cannot be built: %w", repository, merge, err) + return nil, fmt.Errorf("the merged message of %s at %s cannot be built: %w", repository, merge, err) } - return msg.Data, true, nil + f, err := forgemsg.DecodeForge(msg.Data) + if err != nil { + return nil, err + } + if g, ok := groups.Named(of, f.Branch, f.Repository, f.Number); ok { + f.Group, f.Members = g.ID, groups.MembersOf(g) + } + return forgemsg.EncodeForge(f) } -// read is the groups the gate holds with what the bridge kept of them: their ends in performed, and the last ask of -// each in end-asks with the bridge's answer. -func read(merges, performed, asks stdio.KeptState) ([]groups.Row, []string, error) { - keys, err := merges.Keys() - if err != nil { - return nil, nil, fmt.Errorf("the gate's merges cannot be listed: %w", err) - } - raws := map[string][]byte{} - for _, k := range keys { - e, err := merges.Get(k) - if err != nil || e == nil { - continue - } - raws[k] = e.Value - } +// read is the groups the seat mesh-delivery answers with what the bridge kept of them: the merged messages written and +// their ends in performed, and the last ask of each in end-asks with the bridge's answer. +func read(of []seat.Group, performed, asks stdio.KeptState) ([]groups.Row, []string, error) { pkeys, err := performed.Keys() if err != nil { return nil, nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err) } - published, ended := map[string]bool{}, map[string]uint64{} + published, ended := map[string]string{}, map[string]string{} for _, k := range pkeys { + into := ended switch { case strings.HasPrefix(k, "published."): - published[k] = true - case strings.HasPrefix(k, "groups."): - if e, err := performed.Get(k); err == nil && e != nil { - rev, _ := strconv.ParseUint(strings.TrimSpace(string(e.Value)), 10, 64) - ended[strings.TrimPrefix(k, "groups.")] = rev - } + into = published + case !strings.HasPrefix(k, "groups."): + continue + } + if e, err := performed.Get(k); err == nil && e != nil { + into[k] = strings.TrimSpace(string(e.Value)) } } akeys, err := asks.Keys() @@ -224,28 +175,39 @@ func read(merges, performed, asks stdio.KeptState) ([]groups.Row, []string, erro askRaws[k] = e.Value } } - rows, unread := groups.Rows(raws, published, ended, askRaws) + rows, unread := groups.Rows(of, published, ended, askRaws) return rows, unread, nil } -// askEnd writes the end-asks value at key and waits, on a watch of the key, at most bound for the bridge's answer. +// askEnd writes the end-asks value at key and waits at most bound for the bridge's answer. func askEnd(s stdio.KeptState, bound time.Duration) asker { + wait := askState(s, groups.EndAsksBucket, bound, func(v []byte) bool { a, err := groups.DecodeEndAsk(v); return err == nil && a.Done() }) return func(key string, value []byte) (groups.EndAsk, error) { - answered := make(chan groups.EndAsk, 1) + v, err := wait(key, value) + if err != nil { + return groups.EndAsk{}, err + } + return groups.DecodeEndAsk(v) + } +} + +// askState writes an ask's value at key of the state s (its bucket named bucket) and waits, on a watch of the key, at most +// bound for the bridge's answer, a later value that done says is answered. +func askState(s stdio.KeptState, bucket string, bound time.Duration, done func([]byte) bool) func(key string, value []byte) ([]byte, error) { + return func(key string, value []byte) ([]byte, error) { + answered := make(chan []byte, 1) var rev uint64 var mu sync.Mutex take := func(revision uint64, v []byte) { mu.Lock() mine := rev mu.Unlock() - if mine == 0 || revision <= mine { + if mine == 0 || revision <= mine || !done(v) { return } - if a, err := groups.DecodeEndAsk(v); err == nil && a.Done() { - select { - case answered <- a: - default: - } + select { + case answered <- v: + default: } } if err := s.Watch(key, func(c stdio.StateChange) error { @@ -254,11 +216,11 @@ func askEnd(s stdio.KeptState, bound time.Duration) asker { } return nil }); err != nil { - return groups.EndAsk{}, fmt.Errorf("watching %s.%s: %w", groups.EndAsksState, key, err) + return nil, fmt.Errorf("watching %s.%s: %w", bucket, key, err) } r, err := s.Put(key, json.RawMessage(value)) if err != nil { - return groups.EndAsk{}, fmt.Errorf("the ask could not be written to %s: %w", groups.EndAsksBucket, err) + return nil, fmt.Errorf("the ask could not be written to %s: %w", bucket, err) } mu.Lock() rev = r @@ -269,10 +231,10 @@ func askEnd(s stdio.KeptState, bound time.Duration) asker { wait, cancel := context.WithTimeout(context.Background(), bound) // a context's bound, no timer (ADR 0437 §8) defer cancel() select { - case a := <-answered: - return a, nil + case v := <-answered: + return v, nil case <-wait.Done(): - return groups.EndAsk{}, fmt.Errorf("the bridge did not answer within %s: its answer will be in %s.%s", bound, groups.EndAsksBucket, key) + return nil, fmt.Errorf("the bridge did not answer within %s: its answer will be in %s.%s", bound, bucket, key) } } } @@ -283,23 +245,32 @@ func askCaller(_ context.Context, key string, body any) (json.RawMessage, error) } func main() { - merges, performed, asks := stdio.State(gateMergesState), stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState) + performed, asks := stdio.State(groups.PerformedState), stdio.State(groups.EndAsksState) tools := Tools(askEnd(asks, 25*time.Second), time.Now, - func() ([]groups.Row, []string, error) { return read(merges, performed, asks) }, - func(repository, merge string, number int) (json.RawMessage, bool, error) { - keys, err := merges.Keys() + func() ([]groups.Row, []string, error) { + of, err := groups.Read(context.Background(), askCaller) if err != nil { - return nil, false, fmt.Errorf("the gate's merges cannot be listed: %w", err) - } - raws := map[string][]byte{} - for _, k := range keys { - if e, err := merges.Get(k); err == nil && e != nil { - raws[k] = e.Value - } + return nil, nil, err } + return read(of, performed, asks) + }, + func(repository, merge string, number int) (json.RawMessage, error) { ctx := context.Background() - return MergedOf(raws, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), askCaller) + of, err := groups.Read(ctx, askCaller) + if err != nil { + return nil, err + } + return MergedOf(of, repository, merge, number, bridge.TreesVia(ctx, askCaller), bridge.FilesVia(ctx, askCaller), askCaller) }, askCaller) + owedAsks := stdio.State(groups.OwedAsksState) + recorder := meshhandactsclient.Recorder{Ask: stdio.Ask} + tools = append(tools, OwedTools(askState(owedAsks, groups.OwedAsksBucket, 25*time.Second, func(v []byte) bool { + a, err := groups.DecodeOwedAsk(v) + return err == nil && a.Done() + }), func() ([]OwedRow, error) { return performedRows(performed) }, func(what, why string) error { + _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: "owed-dropped"}) + return err + }, time.Now)...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-forge-bridge-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go index 616d23ea..d065eac3 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/main_test.go @@ -20,28 +20,24 @@ import ( "mesh-delivery-forge-bridge/internal/groups" ) -// 1585/T18, T19 (ADR 0633 §4): bridge_groups names each group the gate holds with its members' state and whether its end is -// written (open or ended, with why), narrowed by group and by open; bridge_group_end_again writes an ask naming the group, -// who asked and why, and answers what the bridge answered. +// 1585/T18, T19 (ADR 0633 §4), acceptance 49: bridge_groups names each group the seat mesh-delivery's verb groups answers +// with its members' state and whether its end is written (open or ended, with why), narrowed by group and by open; +// bridge_group_end_again writes an ask naming the group, who asked and why, and answers what the bridge answered. func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) - enc := func(group string, state2 meshdeliveryclient.MergeState, c string) []byte { - v, err := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: group, Ask: "ask", Approved: map[string]string{"novox/a@1": h1, "novox/b@2": h2}, - Pulls: []seat.MergesByGroupValuePullsItem{{Repository: "novox/a", Number: 1, Head: h1, State: string(meshdeliveryclient.MergeMerged), Commit: c1}, - {Repository: "novox/b", Number: 2, Head: h2, State: string(state2), Commit: c}}}) - if err != nil { - t.Fatal(err) - } - return v + grp := func(id string, a, b int, stateB string) seat.Group { + return seat.Group{ID: id, State: "merging", Members: []seat.Member{{Repository: "novox/a", Number: int64(a), Head: h1, State: "merged"}, + {Repository: "novox/b", Number: int64(b), Head: h2, State: stateB}}} } - lone, _ := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "solo", Ask: "ask", Approved: map[string]string{"novox/a@5": h1}, - Pulls: []seat.MergesByGroupValuePullsItem{{Repository: "novox/a", Number: 5, Head: h1, State: string(meshdeliveryclient.MergeMerged), Commit: c1}}}) - merges := map[string][]byte{"ended": enc("ended", meshdeliveryclient.MergeMerged, c2), "ready": enc("ready", meshdeliveryclient.MergeMerged, c2), - "gathered": enc("gathered", meshdeliveryclient.MergeMerging, ""), "solo": lone, "bad": []byte("{")} - performed := map[string]bool{} - for _, c := range []string{c1, c2} { - performed[groups.PublishedKey("novox/a", 1, c)], performed[groups.PublishedKey("novox/b", 2, c)] = true, true + of := []seat.Group{grp("ended", 1, 2, "merged"), grp("ready", 3, 4, "merged"), grp("gathered", 5, 6, "merging"), + {ID: "solo", Members: []seat.Member{{Repository: "novox/a", Number: 7, Head: h1}}}, grp("", 8, 9, "merged")} + published := map[string]string{} + for _, n := range []int{1, 3, 5} { + published[groups.PublishedKey("novox/a", n)] = c1 + } + for _, n := range []int{2, 4} { + published[groups.PublishedKey("novox/b", n)] = c2 } // The state end-asks keeps the last ask of the group ready, answered, and one that does not read. answeredAsk, _ := json.Marshal(groups.EndAsk{V: 1, Group: "ready", Why: "lost", By: "lead@shanks", At: time.Date(2026, 11, 4, 22, 0, 0, 0, time.UTC), @@ -58,7 +54,7 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { } tools := Tools(ask, func() time.Time { return time.Date(2026, 11, 4, 23, 0, 0, 0, time.UTC) }, func() ([]groups.Row, []string, error) { - rows, unread := groups.Rows(merges, performed, map[string]uint64{"ended": 7}, asks) + rows, unread := groups.Rows(of, published, map[string]string{"groups.ended": c2}, asks) return rows, unread, nil }, nil, nil) call := func(name string, args map[string]any) any { @@ -87,22 +83,22 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { for _, r := range out.Groups { m[r.Group] = r } - if len(args) == 0 && strings.Join(out.Unread, " ") != "bad end-asks.gathered" { - t.Errorf("unread = %v, want bad and end-asks.gathered", out.Unread) + if len(args) == 0 && strings.Join(out.Unread, ", ") != "a group with no id, end-asks.gathered" { + t.Errorf("unread = %v, want the group with no id and end-asks.gathered", out.Unread) } return m } all := rowsOf(map[string]any{}) - if len(all) != 3 || !all["ended"].Ended || all["ended"].EndedAt != 7 { + if len(all) != 3 || !all["ended"].Ended || all["ended"].EndedWith != c2 { t.Fatalf("groups = %+v", all) } - if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published { - t.Errorf("ready = %+v, want open, merged and published, its end the thing to say", r) + if r := all["ready"]; !r.Open || !strings.Contains(r.Words, "bridge_group_end_again") || !r.Members[1].Published || r.Members[1].Commit != c2 { + t.Errorf("ready = %+v, want open and published with its merge commit, its end the thing to say", r) } if a := all["ready"].LastEndAsk; a == nil || a.By != "lead@shanks" || a.Why != "lost" || !a.Written || a.Answer == "" || all["ended"].LastEndAsk != nil { t.Errorf("ready's last end ask = %+v, ended's %+v: want ready's ask and its answer, none for ended", a, all["ended"].LastEndAsk) } - if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != string(meshdeliveryclient.MergeMerging) { + if r := all["gathered"]; !r.Open || !strings.Contains(r.Words, "gathered") || r.Members[1].State != "merging" || r.Members[1].Published { t.Errorf("gathered = %+v", r) } if open := rowsOf(map[string]any{"open": true}); len(open) != 2 || open["ended"].Group != "" { @@ -150,54 +146,47 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { } return f } - lone, _ := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "solo", Ask: "a", Approved: map[string]string{"novox/a@5": h1}, - Pulls: []seat.MergesByGroupValuePullsItem{{Repository: "novox/a", Number: 5, Head: h1, State: string(meshdeliveryclient.MergeMerged), Commit: c1}}}) - group, _ := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "feat/g", Ask: "a", Approved: map[string]string{"novox/a@6": h1, "novox/b@7": h2}, - Pulls: []seat.MergesByGroupValuePullsItem{{Repository: "novox/a", Number: 6, Head: h1, State: string(meshdeliveryclient.MergeMerged), Commit: c1}, - {Repository: "novox/b", Number: 7, Head: h2, State: string(meshdeliveryclient.MergeMerging)}}}) + // The seat mesh-delivery's groups: feat/g of novox/a#9 and novox/b#7; a lone pull request is in none. + of := []seat.Group{{ID: "fix/x", Members: []seat.Member{{Repository: "novox/a", Number: 9, Head: h2}, {Repository: "novox/b", Number: 7, Head: h1}}}} for _, c := range []struct { name string - raws map[string][]byte + of []seat.Group repository string merge string number int api bridge.Caller - onForge bool want forgemsg.Forge fail string }{ - {name: "a lone pull request", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c1, want: published("a", 5, h1, "solo", c1)}, - {name: "a member of a group", raws: map[string][]byte{"feat~g": group}, repository: "novox/a", merge: c1, want: func() forgemsg.Forge { - f := published("a", 6, h1, "feat/g", c1) - f.Group, f.Members = "feat/g", []meshdeliveryclient.Head{{Repository: "novox/a", Number: 6, Head: h1}, {Repository: "novox/b", Number: 7, Head: h2}} - return f - }()}, - {name: "a merge the gate does not hold", raws: map[string][]byte{"solo": lone, "bad": []byte("{")}, repository: "novox/a", merge: c2, fail: "hold no merge"}, - // Issue 1966: a merge made on the forge (a recovery merge) has no merge in the gate; the bridge reads the pull request - // from the forge, by number or by the merge commit, and checks its merge commit. - {name: "a merge made on the forge, by number", raws: map[string][]byte{"solo": lone}, repository: "novox/a", merge: c2, number: 9, onForge: true, + {name: "a lone pull request, by number", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, - {name: "a merge made on the forge, by the merge commit", raws: map[string][]byte{}, repository: "novox/a", merge: c2, onForge: true, + {name: "a lone pull request, by the merge commit", repository: "novox/a", merge: c2, api: forgeAnswers(t, 0, c2, "merged", "fix/x", h2, c2, "main"), want: published("a", 9, h2, "fix/x", c2)}, - {name: "a forge pull request merged at another commit", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a member of a group", of: of, repository: "novox/a", merge: c2, + api: forgeAnswers(t, 0, c2, "merged", "fix/x", h2, c2, "main"), want: func() forgemsg.Forge { + f := published("a", 9, h2, "fix/x", c2) + f.Group, f.Members = "fix/x", []meshdeliveryclient.Head{{Repository: "novox/a", Number: 9, Head: h2}, {Repository: "novox/b", Number: 7, Head: h1}} + return f + }()}, + {name: "a forge pull request merged at another commit", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c1, "main"), fail: "not at " + c2}, - {name: "a forge pull request not merged", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a forge pull request not merged", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "open", "fix/x", h2, "", "main"), fail: "not merged"}, - {name: "a forge pull request merged into another branch", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a forge pull request merged into another branch", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "dev"), fail: "not the trunk"}, - {name: "a forge pull request with no head branch", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a forge pull request with no head branch", repository: "novox/a", merge: c2, number: 9, api: forgeAnswers(t, 9, "", "merged", "", h2, c2, "main"), fail: "no head branch"}, - {name: "a seat answer that says an error", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a seat answer that says an error", repository: "novox/a", merge: c2, number: 9, api: answering(gitfixture.Given(t, "pull answers an error, never none", "verb:pull.answer")), fail: "holds no pull request"}, - {name: "a seat answer with no pull request", raws: map[string][]byte{}, repository: "novox/a", merge: c2, number: 9, + {name: "a seat answer with no pull request", repository: "novox/a", merge: c2, number: 9, api: answering(json.RawMessage(`{}`)), fail: "carries nothing"}, - {name: "a forge that does not answer", raws: map[string][]byte{}, repository: "novox/a", merge: c2, + {name: "a forge that does not answer", repository: "novox/a", merge: c2, api: func(context.Context, string, any) (json.RawMessage, error) { return nil, errors.New("the forge is down") }, fail: "the forge is down"}, } { t.Run(c.name, func(t *testing.T) { - raw, onForge, err := MergedOf(c.raws, c.repository, c.merge, c.number, trees, files, c.api) + raw, err := MergedOf(c.of, c.repository, c.merge, c.number, trees, files, c.api) if c.fail != "" { if err == nil || !strings.Contains(err.Error(), c.fail) { t.Fatalf("err %v, want %q", err, c.fail) @@ -205,20 +194,67 @@ func TestTheBridgeToolsReadTheGroupsAndAskTheEndAgain(t *testing.T) { return } want, _ := forgemsg.EncodeForge(c.want) - if err == nil && onForge != c.onForge { - t.Fatalf("made on the forge %v, want %v", onForge, c.onForge) - } if err != nil || !bytes.Equal(raw, want) { t.Fatalf("err %v, answer\n%s\nwant the published message\n%s", err, raw, want) } }) } - if _, _, err := MergedOf(map[string][]byte{"solo": lone}, "novox/a", c1, 0, func(string, string, string) (string, map[string]string, error) { + if _, err := MergedOf(nil, "novox/a", c2, 9, func(string, string, string) (string, map[string]string, error) { return "", nil, errors.New("the forge does not answer") - }, files, nil); err == nil || !strings.Contains(err.Error(), "does not answer") { + }, files, forgeAnswers(t, 9, "", "merged", "fix/x", h2, c2, "main")); err == nil || !strings.Contains(err.Error(), "does not answer") { t.Errorf("a forge that does not answer: %v", err) } }) + + // Design 47 *What the gate owes the forge*: bridge_owed reads performed and asks the backlog; bridge_owed_again asks the + // act again; bridge_owed_drop writes the hand-act log first and drops nothing it could not record. + t.Run("the owed tools", func(t *testing.T) { + at := time.Date(2026, 11, 9, 21, 0, 0, 0, time.UTC) + var asked []groups.OwedAsk + ask := func(key string, value []byte) ([]byte, error) { + a, err := groups.DecodeOwedAsk(value) + if err != nil { + return nil, err + } + asked = append(asked, a) + a.Answered, a.Outcome, a.Answer = at, forgemsg.OwedPerformed, "said as owed.ended: performed" + if a.Kind == groups.OwedBacklog { + a.Backlog = []groups.Backlog{{Stream: seat.StreamOwed, Waiting: 1}} + } + return json.Marshal(a) + } + rows := []OwedRow{{Of: seat.Seat, ID: "a", Outcome: forgemsg.OwedPerformed, At: at}, {Of: "mesh-delivery-checker", ID: "b", Outcome: forgemsg.OwedRefused, At: at}} + var recorded []string + recordErr := error(nil) + record := func(what, why string) error { recorded = append(recorded, what+": "+why); return recordErr } + tools := map[string]func(map[string]any) (any, error){} + for _, tl := range OwedTools(ask, func() ([]OwedRow, error) { return rows, nil }, record, func() time.Time { return at }) { + tools[tl.Name] = tl.Run + } + out, err := tools[OwedTool](map[string]any{"of": seat.Seat}) + if m, _ := out.(map[string]any); err != nil || len(m["performed"].([]OwedRow)) != 1 || len(m["backlog"].([]groups.Backlog)) != 1 { + t.Errorf("bridge_owed: %v %v", out, err) + } + if out, err := tools[OwedAgainTool](map[string]any{"of": seat.Seat, "id": "a"}); err != nil || out.(map[string]any)["outcome"] != forgemsg.OwedPerformed || + asked[len(asked)-1].Kind != groups.OwedAgain { + t.Errorf("bridge_owed_again: %v %v", out, err) + } + if _, err := tools[OwedDropTool](map[string]any{"of": seat.Seat, "id": "a"}); err == nil || len(recorded) != 0 { + t.Errorf("a drop with no why: %v, recorded %v", err, recorded) + } + n := len(asked) + recordErr = errors.New("mesh-hand-acts has no holder that answers") + if _, err := tools[OwedDropTool](map[string]any{"of": seat.Seat, "id": "a", "why": "merged by hand"}); err == nil || len(asked) != n { + t.Errorf("a drop whose hand act was not recorded: %v, asked %d", err, len(asked)-n) + } + recordErr = nil + if _, err := tools[OwedDropTool](map[string]any{"of": seat.Seat, "id": "a", "why": "merged by hand"}); err != nil || asked[len(asked)-1].Why != "merged by hand" { + t.Errorf("bridge_owed_drop: %v %+v", err, asked) + } + if _, _, err := groups.NewOwedAsk(groups.OwedAgain, "", "a", "", "", at); err == nil { + t.Error("an ask naming no seat was taken") + } + }) } // forgeAnswers is the seat git's verb pull for one pull request, asked by number (when number is given) or by the merge diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go new file mode 100644 index 00000000..2c7459b0 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge-tools/owed.go @@ -0,0 +1,132 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + + "mesh-delivery-forge-bridge/internal/groups" +) + +// The tools of what the bridge performs of what is owed to the forge (novox/hq design 47 *What the gate owes the forge*). +const ( + OwedTool = "bridge_owed" + OwedAgainTool = "bridge_owed_again" + OwedDropTool = "bridge_owed_drop" +) + +// OwedRow is one act the bridge performed, as the state performed keeps it: the outcome said as owed.ended. +type OwedRow = forgemsg.OwedEnded + +// performedRows reads every act the state performed holds; a key that is not an act (a group end's bookkeeping) is left. +func performedRows(s stdio.KeptState) ([]OwedRow, error) { + keys, err := s.Keys() + if err != nil { + return nil, fmt.Errorf("the bridge's performed state cannot be listed: %w", err) + } + var rows []OwedRow + for _, k := range keys { + e, err := s.Get(k) + if err != nil || e == nil { + continue + } + var p struct { + Ended OwedRow `json:"ended"` + } + if json.Unmarshal(e.Value, &p) == nil && p.Ended.Of != "" { + rows = append(rows, p.Ended) + } + } + sort.Slice(rows, func(i, j int) bool { return rows[i].At.After(rows[j].At) }) + return rows, nil +} + +// OwedTools are bridge_owed, bridge_owed_again and bridge_owed_drop: ask writes an owed-asks value and waits for the +// bridge's answer; performed reads the state performed; record writes a hand act in the seat mesh-hand-acts' log before +// a drop is asked, so a drop is never made without its record. +func OwedTools(ask func(key string, value []byte) ([]byte, error), performed func() ([]OwedRow, error), record func(what, why string) error, now func() time.Time) []stdio.Tool { + str := func(args map[string]any, k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + act := map[string]any{ + "of": map[string]any{"type": "string", "description": "the seat that said the act: mesh-delivery (the gate) or mesh-delivery-checker"}, + "id": map[string]any{"type": "string", "description": "the act's id, as owed and owed.ended say it"}} + asked := func(kind string, args map[string]any) (groups.OwedAsk, error) { + key, value, err := groups.NewOwedAsk(kind, str(args, "of"), str(args, "id"), str(args, "why"), stdio.CallerOf(args), now()) + if err != nil { + return groups.OwedAsk{}, err + } + v, err := ask(key, value) + if err != nil { + return groups.OwedAsk{}, err + } + return groups.DecodeOwedAsk(v) + } + withWhy := map[string]any{"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}} + for k, v := range act { + withWhy[k] = v + } + return []stdio.Tool{ + {Name: OwedTool, + Description: "What the forge bridge performed of what the gate and the checker owe the forge (design 47): each act it said as " + + "owed.ended, newest first, with its seat, id, pull request, outcome (performed, refused, could not be performed, " + + "undecodable, dropped), the forge's words and when, from its state performed (kept 7 days); and the backlog of its " + + "durable on each seat's stream owed: the last event applied, the events waiting and held, the age of the oldest. " + + "of or id narrow it. Read-only.", + Input: map[string]any{"type": "object", "properties": act}, + Run: func(args map[string]any) (any, error) { + rows, err := performed() + if err != nil { + return nil, err + } + kept := []OwedRow{} + for _, r := range rows { + if (str(args, "of") == "" || r.Of == str(args, "of")) && (str(args, "id") == "" || r.ID == str(args, "id")) { + kept = append(kept, r) + } + } + out := map[string]any{"performed": kept} + if a, err := asked(groups.OwedBacklog, args); err != nil { + out["backlog-error"] = err.Error() + } else { + out["backlog"] = a.Backlog + } + return out, nil + }}, + {Name: OwedAgainTool, + Description: "Perform one act owed to the forge again now, by hand, whatever its last outcome (a refusal the forge no longer " + + "makes, an act that could not be performed): the bridge calls the seat git's verb of the act under its id (safe to " + + "repeat) and says its outcome as owed.ended. The act is the one the bridge kept, or, for the gate's, the seat " + + "mesh-delivery's groups. Answers the outcome.", + Input: map[string]any{"type": "object", "required": []string{"of", "id"}, "properties": act}, + Run: func(args map[string]any) (any, error) { + a, err := asked(groups.OwedAgain, args) + if err != nil { + return nil, err + } + return map[string]any{"of": a.Of, "id": a.ID, "outcome": a.Outcome, "answer": a.Answer}, nil + }}, + {Name: OwedDropTool, + Description: "Drop one act owed to the forge, by hand, with why: written first in the hand-act log of the seat mesh-hand-acts, " + + "then said by the bridge as owed.ended with the outcome dropped, so its owner records it answered and stops saying " + + "it again. Nothing is done at the forge. Answers the outcome said.", + Input: map[string]any{"type": "object", "required": []string{"of", "id", "why"}, "properties": withWhy}, + Run: func(args map[string]any) (any, error) { + if str(args, "why") == "" { + return nil, fmt.Errorf("a drop says why") + } + if err := record(fmt.Sprintf("dropped the act %s that %s owes the forge", str(args, "id"), str(args, "of")), str(args, "why")); err != nil { + return nil, err + } + a, err := asked(groups.OwedDrop, args) + if err != nil { + return nil, err + } + return map[string]any{"of": a.Of, "id": a.ID, "outcome": a.Outcome, "answer": a.Answer}, nil + }}, + } +} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go index a005d465..d0120eb9 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go @@ -3,142 +3,10 @@ package main import ( "context" "errors" - "fmt" - "log" - "strconv" "github.com/nats-io/nats.go/jetstream" - - "mesh-delivery-forge-bridge/internal/bridge" ) -// performedBucket is the module's state bucket performed, `_` as heads is: for each key of the gate's -// buckets the bridge has acted on, the revision it acted at (manifest state entry performed, novox/hq ADR 0437 §8). -// Its keys are `owed.` and `merges.`; its value is the revision in decimal. -const performedBucket = forgeBridgeModule + "_performed" - -// kvRevisions is the bucket performed as the performer's revisions. -type kvRevisions struct{ kv jetstream.KeyValue } - -func (s kvRevisions) Get(ctx context.Context, key string) (uint64, bool, error) { - e, err := s.kv.Get(ctx, key) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return 0, false, nil - } - if err != nil { - return 0, false, err - } - rev, err := strconv.ParseUint(string(e.Value()), 10, 64) - if err != nil { - return 0, false, fmt.Errorf("%s holds %q, which is no revision: %w", key, e.Value(), err) - } - return rev, true, nil -} - -func (s kvRevisions) Put(ctx context.Context, key string, rev uint64) error { - _, err := s.kv.PutString(ctx, key, strconv.FormatUint(rev, 10)) - return err -} - -// revisions is what the performer keeps of the revisions it acted at. -type revisions interface { - Get(ctx context.Context, key string) (rev uint64, found bool, err error) - Put(ctx context.Context, key string, rev uint64) error -} - -// performer does on the forge what the gate's keys ask (novox/hq ADR 0437 §8): one act for each revision of an owed -// key, one merge for each revision of a merges key. A refusal is written on FORGE and the act is not tried again until -// the key has a new revision; the performer holds no timer and serves no request. -type performer struct { - call caller - pub publisher - done revisions - logger *log.Logger -} - -// seen says whether the revision of the key was acted on already. A revision that cannot be read is not seen, so the -// act is done: a status set twice is the same status. -func (p *performer) seen(ctx context.Context, doneKey string, rev uint64) bool { - kept, found, err := p.done.Get(ctx, doneKey) - if err != nil { - p.logger.Printf("%s: the revision acted at could not be read, so the act is done again: %v", doneKey, err) - return false - } - return found && kept >= rev -} - -func (p *performer) keep(ctx context.Context, doneKey string, rev uint64) { - if err := p.done.Put(ctx, doneKey, rev); err != nil { - p.logger.Printf("%s: revision %d was acted on but could not be kept, so a restart acts on it again: %v", doneKey, rev, err) - } -} - -// refuse writes the refused message of an act on FORGE; it reports whether it was written. The message id makes a -// second write of one revision's refusal the same message to the stream. -func (p *performer) refuse(ctx context.Context, doneKey string, rev uint64, a bridge.Act, words string) bool { - msg, err := bridge.RefusedMessage(a, words) - if err != nil { - p.logger.Printf("%s: the forge refused %s (%s) and the refusal cannot be said on FORGE: %v", doneKey, a.Name, words, err) - return false - } - id := fmt.Sprintf("refused.%s.%d", doneKey, rev) - if _, err := p.pub.Publish(ctx, forgeSubject(msg.Event), msg.Data, jetstream.WithMsgID(id)); err != nil { - p.logger.Printf("%s: the forge refused %s (%s) and FORGE could not be written: %v", doneKey, a.Name, words, err) - return false - } - return true -} - -// owed performs the act an owed key of the gate asks at revision rev, once. -func (p *performer) owed(ctx context.Context, key string, rev uint64, value []byte) { - p.perform(ctx, "owed."+key, key, rev, value) -} - -// perform does the act of an owed key once for the revision, under the bookkeeping key doneKey. -func (p *performer) perform(ctx context.Context, doneKey, key string, rev uint64, value []byte) { - if p.seen(ctx, doneKey, rev) { - return - } - a, err := bridge.OwedAct(key, value) - if err != nil { - p.logger.Printf("%s at revision %d is not an act the bridge can perform, and is left: %v", key, rev, err) - p.keep(ctx, doneKey, rev) - return - } - if _, err := p.call(ctx, a.Call.Tool, a.Call.Args); err != nil { - p.logger.Printf("%s at revision %d: %s refused: %v", key, rev, a.Call.Tool, err) - if !p.refuse(ctx, doneKey, rev, a, err.Error()) { - return - } - } - p.keep(ctx, doneKey, rev) -} - -// merges performs the merges a merges key hands the bridge at revision rev, once: each pull request in the state -// merging, in the order the value names them, stopping at the first refusal (novox/hq ADR 0441 §3). -func (p *performer) merges(ctx context.Context, key string, rev uint64, value []byte) { - doneKey := "merges." + key - if p.seen(ctx, doneKey, rev) { - return - } - acts, err := bridge.MergeActs(value) - if err != nil { - p.logger.Printf("%s at revision %d cannot be merged, and is left: %v", key, rev, err) - p.keep(ctx, doneKey, rev) - return - } - for _, a := range acts { - if _, err := p.call(ctx, a.Call.Tool, a.Call.Args); err != nil { - p.logger.Printf("%s at revision %d: the merge of %s#%d refused: %v", key, rev, a.Repository, a.Number, err) - if !p.refuse(ctx, doneKey, rev, a, err.Error()) { - return - } - break - } - } - p.keep(ctx, doneKey, rev) -} - // watch hands each key of a bucket, with the revision it has now and then each new one, to react until ctx ends or the // watch fails. A deleted key is not an act. func watch(ctx context.Context, kv jetstream.KeyValue, react func(key string, rev uint64, value []byte)) error { diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go index 7fced163..942df8cd 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go @@ -8,10 +8,15 @@ import ( "log" "strings" "testing" + "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + + "mesh-delivery-forge-bridge/internal/gitfixture" + "mesh-delivery-forge-bridge/internal/groups" ) const ( @@ -19,195 +24,231 @@ const ( headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" ) -type memRevisions map[string]uint64 +type memStrs map[string]string -func (m memRevisions) Get(_ context.Context, key string) (uint64, bool, error) { +func (m memStrs) Get(_ context.Context, key string) (string, bool, error) { r, ok := m[key] return r, ok, nil } -func (m memRevisions) Put(_ context.Context, key string, rev uint64) error { m[key] = rev; return nil } +func (m memStrs) Put(_ context.Context, key, v string) error { m[key] = v; return nil } -type tcall struct { - key string - body string +type memPerformed map[string][]byte + +func (m memPerformed) Get(_ context.Context, key string) ([]byte, bool, error) { + v, ok := m[key] + return v, ok, nil +} +func (m memPerformed) Put(_ context.Context, key string, v []byte) error { m[key] = v; return nil } + +// seatFake answers each seat verb from answers, by its key; one absent is not answered (an error, as a bound passed). +type seatFake struct { + calls []string + answers map[string]json.RawMessage } -// forgeFake records the calls and answers each from refuse, a map from a tool to its refusal. -type forgeFake struct { - calls []tcall - refuse map[string]string -} - -func (f *forgeFake) call(_ context.Context, key string, body any) (json.RawMessage, error) { - b, _ := json.Marshal(body) - f.calls = append(f.calls, tcall{key, string(b)}) - if w, ok := f.refuse[key]; ok { - return nil, errors.New(w) +func (f *seatFake) call(_ context.Context, key string, _ any) (json.RawMessage, error) { + f.calls = append(f.calls, key) + if a, ok := f.answers[key]; ok { + return a, nil } - return json.RawMessage(`{}`), nil + return nil, errors.New("nats: timeout") } -func newPerformer(f *forgeFake) (*performer, *fakePub, memRevisions, *bytes.Buffer) { - pub, done, logs := &fakePub{}, memRevisions{}, &bytes.Buffer{} - return &performer{call: f.call, pub: pub, done: done, logger: log.New(logs, "", 0)}, pub, done, logs +func newOwing(f *seatFake) (*owing, *fakePub, memPerformed) { + pub, kept := &fakePub{}, memPerformed{} + at := time.Date(2026, 11, 9, 21, 0, 0, 0, time.UTC) + return &owing{call: f.call, pub: pub, performed: kept, now: func() time.Time { return at }, logger: log.New(&bytes.Buffer{}, "", 0)}, pub, kept } -// The values are the SDK's own encoding (mesh-sdk go/delivery EncodeOwed, EncodeMerge), the keys its OwedKey. -func owedKV(t *testing.T, repository string, number int, act string, o seat.OwedByRepositoryNumberHeadActValue) (string, []byte) { - t.Helper() - key, err := meshdeliveryclient.OwedKey(repository, number, headA, act) - if err != nil { - t.Fatal(err) - } - v, err := meshdeliveryclient.EncodeOwed(o) - if err != nil { - t.Fatal(err) - } - return key, v -} - -func TestAnOwedStatusIsSetOnceForARevision(t *testing.T) { // 1341/AC1 +// Design 47 *What the gate owes the forge*: an owed act is the seat git's verb under its id, its outcome said as +// owed.ended on the stream forge before it is kept in performed, an act it cannot read said undecodable; and at its start +// the bridge performs from the seat's groups every act said and not answered. +func TestWhatIsOwedIsPerformedSaidAndKept(t *testing.T) { t.Run("the names are the manifest's", theNamesAreTheManifests) - f := &forgeFake{} - p, _, _, _ := newPerformer(f) - key, v := owedKV(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate", - seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: "mesh/merge-gate", State: "success", Text: "all green", Target: "https://forge.example/p/12"}) - p.owed(context.Background(), key, 7, v) - if len(f.calls) != 1 || f.calls[0].key != "gitea.gitea_commit_status" || - f.calls[0].body != `{"context":"mesh/merge-gate","description":"all green","owner":"novox","repo":"mesh-delivery","sha":"`+headA+`","state":"success","target_url":"https://forge.example/p/12"}` { - t.Fatalf("calls = %+v", f.calls) - } - p.owed(context.Background(), key, 7, v) // the same revision seen again (a restart's replay) - if len(f.calls) != 1 { - t.Errorf("the same revision called the tool again: %+v", f.calls) - } - p.owed(context.Background(), key, 8, v) // a new revision of the key is a new act - if len(f.calls) != 2 { - t.Errorf("a new revision was not performed: %+v", f.calls) - } -} - -func TestViewNoteAndCommentEachCallTheirToolOnce(t *testing.T) { // 1341/AC2 - f := &forgeFake{} - p, _, _, _ := newPerformer(f) + gate := func(name string) []byte { return gitfixture.Seat(t, seat.Seat, "events", name, "event:owed")[0] } + answer := func(verb, name string) json.RawMessage { return gitfixture.Given(t, name, "verb:"+verb+".answer") } for _, c := range []struct { - act string - o seat.OwedByRepositoryNumberHeadActValue + name, of string + data []byte + answers map[string]json.RawMessage + pubErr error + call string + outcome string + kept bool + err string }{ - {"view", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedView, Text: "the view"}}, - {"note", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedNote, Text: "a line", Ref: "mesh-plan"}}, - {"comment.1", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "hello"}}, + {"a merge performed", seat.Seat, gate("owed is said, a merge"), + map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers")}, nil, + "seat:git.merge-pull", forgemsg.OwedPerformed, true, ""}, + {"a status refused", seat.Seat, gate("owed is said, a status"), + map[string]json.RawMessage{"seat:git.status": answer("status", "status answers an error, never none")}, nil, + "seat:git.status", forgemsg.OwedRefused, true, ""}, + {"the checker's status not answered", checker.Seat, gitfixture.Seat(t, checker.Seat, "events", "owed is said", "event:owed")[0], + nil, nil, "seat:git.status", forgemsg.OwedCouldNotBePerformed, true, ""}, + {"an act it cannot read", seat.Seat, []byte(`{`), nil, nil, "", forgemsg.OwedUndecodable, false, ""}, + {"a seat that says no owed act", "mesh-wants", gate("owed is said, a merge"), nil, nil, "", forgemsg.OwedUndecodable, false, ""}, + {"the stream forge refusing the outcome", seat.Seat, gate("owed is said, a merge"), + map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers")}, errors.New("no responders"), + "seat:git.merge-pull", "", false, "could not be said"}, } { - key, v := owedKV(t, "novox/mesh-delivery", 12, c.act, c.o) - p.owed(context.Background(), key, 1, v) - p.owed(context.Background(), key, 1, v) + f := &seatFake{answers: c.answers} + o, pub, kept := newOwing(f) + pub.err = c.pubErr + _, err := o.heard(context.Background(), c.of, c.data) + if c.err != "" { + if err == nil || !strings.Contains(err.Error(), c.err) || len(kept) != 0 { + t.Errorf("%s: %v, kept %d; want an error naming %q and nothing kept", c.name, err, len(kept), c.err) + } + continue + } + if err != nil || len(pub.data) != 1 || pub.subjects[0] != forgeSubject(forgemsg.OwedEndedEvent) { + t.Fatalf("%s: %v, said %v; want one owed.ended", c.name, err, pub.subjects) + } + e, err := forgemsg.DecodeOwedEnded(pub.data[0]) + if err != nil || e.Outcome != c.outcome || e.Of != c.of { + t.Errorf("%s: said %+v %v, want %s of %s", c.name, e, err, c.outcome, c.of) + } + if (c.call == "") != (len(f.calls) == 0) || (c.call != "" && f.calls[0] != c.call) { + t.Errorf("%s: called %v, want %q", c.name, f.calls, c.call) + } + if v, ok := kept[performedKey(c.of, e.ID)]; ok != c.kept { + t.Errorf("%s: kept %v (%s), want %v", c.name, ok, v, c.kept) + } else if ok { + var p Performed + if err := json.Unmarshal(v, &p); err != nil || p.Ended.Outcome != c.outcome || !json.Valid(p.Owed) { + t.Errorf("%s: kept %s, want the outcome and the owed event", c.name, v) + } + } } - var tools []string - for _, c := range f.calls { - tools = append(tools, c.key) - } - if got := strings.Join(tools, " "); got != "gitea.gitea_delivery_view gitea.gitea_note_append gitea.gitea_add_comment" { - t.Errorf("calls = %s", got) - } -} -func TestARefusedActIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1341/AC3 - f := &forgeFake{refuse: map[string]string{"gitea.gitea_add_comment": "the pull request is locked"}} - p, pub, _, _ := newPerformer(f) - key, v := owedKV(t, "novox/mesh-delivery", 12, "comment.1", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "hello"}) - p.owed(context.Background(), key, 3, v) - p.owed(context.Background(), key, 3, v) - if len(f.calls) != 1 { - t.Errorf("the refused act was tried %d times at one revision", len(f.calls)) - } - if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.refused" { - t.Fatalf("published on %v", pub.subjects) - } - m, err := forgemsg.DecodeForge(pub.data[0]) - if err != nil || m.Kind != forgemsg.ForgeRefused || m.Act != "comment.1" || m.Number != 12 || m.Head != headA || m.Words != "the pull request is locked" { - t.Errorf("message %+v err %v", m, err) - } - f.refuse = nil - p.owed(context.Background(), key, 4, v) // only a new revision tries again - if len(f.calls) != 2 { - t.Errorf("a new revision was not tried: %d calls", len(f.calls)) - } -} + // Design 47 *The switch carries what is owed*: an act the gate switched from its old state is performed only when the old + // bridge's own bookkeeping does not name its key at its revision or later; one it names is said performed, not done. + t.Run("a switched act the old bridge performed", func(t *testing.T) { + var owed map[string]any + if err := json.Unmarshal(gate("owed is said, a merge"), &owed); err != nil { + t.Fatal(err) + } + id, err := meshdeliveryclient.SwitchedActID(meshdeliveryclient.SwitchedMerges, "feat~x", 7) + if err != nil { + t.Fatal(err) + } + owed["id"] = id + data, _ := json.Marshal(owed) + for _, c := range []struct { + name string + kept string // the old bridge's merges.feat~x, "" for none + called bool + }{ + {"kept at its revision", "7", false}, + {"kept at a later revision", "9", false}, + {"kept at an earlier revision", "6", true}, + {"not kept", "", true}, + } { + f := &seatFake{answers: map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers")}} + o, pub, kept := newOwing(f) + if c.kept != "" { + kept["merges.feat~x"] = []byte(c.kept) + } + if _, err := o.heard(context.Background(), seat.Seat, data); err != nil || len(pub.data) != 1 { + t.Fatalf("%s: %v, said %d", c.name, err, len(pub.data)) + } + e, _ := forgemsg.DecodeOwedEnded(pub.data[0]) + if (len(f.calls) == 1) != c.called || e.Outcome != forgemsg.OwedPerformed || e.ID != id { + t.Errorf("%s: called %v, said %+v; want called %v and performed", c.name, f.calls, e, c.called) + } + if _, ok := kept[performedKey(seat.Seat, id)]; !ok { + t.Errorf("%s: the outcome is not kept under the act's id", c.name) + } + } + }) -func TestAnOwedValueThatCannotBeReadIsLoggedAndNotRetried(t *testing.T) { - f := &forgeFake{} - p, pub, done, logs := newPerformer(f) - p.owed(context.Background(), "not-a-key", 2, []byte(`{}`)) - if len(f.calls) != 0 || len(pub.subjects) != 0 { - t.Errorf("calls %v published %v", f.calls, pub.subjects) - } - if !strings.Contains(logs.String(), "not-a-key") { - t.Errorf("log %q does not name the key", logs.String()) - } - if rev, ok := done["owed.not-a-key"]; !ok || rev != 2 { - t.Errorf("revision kept = %v %v", rev, ok) - } -} + t.Run("heard from a seat's stream owed", func(t *testing.T) { + f := &seatFake{answers: map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers")}} + o, pub, _ := newOwing(f) + m := &fakeMsg{subject: "mesh.seat.mesh-delivery.event.owed.novox/mesh-sdk.178", data: gate("owed is said, a merge")} + o.hear(context.Background(), seat.Seat, m) + again := &fakeMsg{subject: m.subject, data: m.data} + o.hear(context.Background(), seat.Seat, again) + if m.acked != 1 || again.acked != 1 || len(f.calls) != 1 || len(pub.data) != 1 { + t.Errorf("an act said twice: acked %d and %d, called %v, said %d; want performed and said once, both acknowledged", + m.acked, again.acked, f.calls, len(pub.data)) + } + pub.err = errors.New("no responders") + lost := &fakeMsg{subject: m.subject, data: gate("owed is said, a status")} + o.hear(context.Background(), seat.Seat, lost) + if lost.acked != 0 || len(lost.nak) != 1 { + t.Errorf("an outcome not said: acked %d, nak %v; want it heard again", lost.acked, lost.nak) + } + }) -func mergeKV(t *testing.T, m seat.MergesByGroupValue) []byte { - t.Helper() - v, err := meshdeliveryclient.EncodeMerge(m) - if err != nil { - t.Fatal(err) - } - return v -} + t.Run("asked again, dropped, and its backlog read by hand", func(t *testing.T) { + groupsAnswer := gitfixture.Seat(t, seat.Seat, "verbs", "groups answers a group with its members and acts", "verb:groups.answer")[0] + f := &seatFake{answers: map[string]json.RawMessage{"seat:git.merge-pull": answer("merge-pull", "merge-pull answers"), + "seat:mesh-delivery.groups": groupsAnswer, "seat:git.status": answer("status", "status answers")}} + o, pub, kept := newOwing(f) + merge := "merge.feat=47-G.0123456789ab.novox/mesh-sdk#178" + if _, err := o.heard(context.Background(), seat.Seat, gate("owed is said, a merge")); err != nil { + t.Fatal(err) + } + backlog := func(context.Context) []groups.Backlog { return []groups.Backlog{{Stream: seat.StreamOwed, Waiting: 2}} } + for _, c := range []struct { + name string + ask groups.OwedAsk + calls, said int + outcome, answers string + }{ + {"again, from performed", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: merge}, 2, 2, forgemsg.OwedPerformed, "performed"}, + // The fixture's status is in the groups only: the bridge never heard it. + {"again, from the groups", groups.OwedAsk{Kind: groups.OwedAgain, Of: seat.Seat, ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate"}, 4, 3, forgemsg.OwedPerformed, "performed"}, + {"dropped", groups.OwedAsk{Kind: groups.OwedDrop, Of: seat.Seat, ID: merge, Why: "merged by hand", By: "lead@shanks"}, 4, 4, forgemsg.OwedDropped, "merged by hand"}, + {"again, an act nobody holds", groups.OwedAsk{Kind: groups.OwedAgain, Of: checker.Seat, ID: "x"}, 4, 4, "", "checker says"}, + {"the backlog", groups.OwedAsk{Kind: groups.OwedBacklog}, 4, 4, "", "backlog"}, + } { + a := o.answer(context.Background(), c.ask, backlog) + if !a.Done() || a.Outcome != c.outcome || !strings.Contains(a.Answer, c.answers) || len(f.calls) != c.calls || len(pub.data) != c.said { + t.Errorf("%s: %+v, called %v, said %d; want %q naming %q, %d calls, %d said", c.name, a, f.calls, len(pub.data), c.outcome, c.answers, c.calls, c.said) + } + if c.ask.Kind == groups.OwedBacklog && (len(a.Backlog) != 1 || a.Backlog[0].Waiting != 2) { + t.Errorf("%s: %+v", c.name, a.Backlog) + } + } + var p Performed + if err := json.Unmarshal(kept[performedKey(seat.Seat, merge)], &p); err != nil || p.Ended.Outcome != forgemsg.OwedDropped { + t.Errorf("performed after the drop: %+v %v", p, err) + } + }) -func twoMembers(a, b meshdeliveryclient.MergeState) seat.MergesByGroupValue { - return seat.MergesByGroupValue{Group: "g", Ask: "merge.g.0123456789ab", - Approved: map[string]string{meshdeliveryclient.ApprovedKey("novox/mesh-sdk", 4): headB, meshdeliveryclient.ApprovedKey("novox/mesh-delivery", 12): headA}, - Pulls: []seat.MergesByGroupValuePullsItem{ - {Repository: "novox/mesh-delivery", Number: 12, Head: headA, State: string(a), Commit: map[bool]string{true: "c1"}[a == meshdeliveryclient.MergeMerged]}, - {Repository: "novox/mesh-sdk", Number: 4, Head: headB, State: string(b), Commit: map[bool]string{true: "c2"}[b == meshdeliveryclient.MergeMerged]}, - }} -} - -func TestAMergeRevisionMergesOnlyTheMemberHanded(t *testing.T) { // 1342/AC1 - f := &forgeFake{} - p, _, _, _ := newPerformer(f) - m := twoMembers(meshdeliveryclient.MergeMerging, meshdeliveryclient.MergeMerged) - p.merges(context.Background(), "g", 5, mergeKV(t, m)) - want := `{"approved_heads":["novox/mesh-delivery@` + headA + `","novox/mesh-sdk@` + headB + `"],"head":"` + headA + `","number":12,"owner":"novox","repo":"mesh-delivery"}` - if len(f.calls) != 1 || f.calls[0].key != "gitea.gitea_merge_pull_request" || f.calls[0].body != want { - t.Fatalf("calls = %+v", f.calls) - } - p.merges(context.Background(), "g", 5, mergeKV(t, m)) - if len(f.calls) != 1 { - t.Errorf("a revision merged twice: %+v", f.calls) - } -} - -func TestARefusedMergeIsWrittenOnForgeOnceAndNotRetried(t *testing.T) { // 1342/AC2 - f := &forgeFake{refuse: map[string]string{"gitea.gitea_merge_pull_request": "novox/mesh-delivery#12 is not merged: its head moved"}} - p, pub, _, _ := newPerformer(f) - v := mergeKV(t, twoMembers(meshdeliveryclient.MergeMerging, meshdeliveryclient.MergeMerged)) - p.merges(context.Background(), "g", 5, v) - p.merges(context.Background(), "g", 5, v) - if len(f.calls) != 1 || len(pub.data) != 1 { - t.Fatalf("calls %d messages %d, want one each", len(f.calls), len(pub.data)) - } - m, err := forgemsg.DecodeForge(pub.data[0]) - if err != nil || m.Kind != forgemsg.ForgeRefused || m.Repository != "novox/mesh-delivery" || m.Number != 12 || m.Head != headA || - m.Act != "merge" || m.Words != "novox/mesh-delivery#12 is not merged: its head moved" { - t.Errorf("message %+v err %v", m, err) - } - f.refuse = nil - p.merges(context.Background(), "g", 6, v) - if len(f.calls) != 2 { - t.Errorf("a new revision was not tried: %d calls", len(f.calls)) - } -} - -func TestAMergeWithTwoHandedStopsAtTheFirstRefusal(t *testing.T) { // ADR 0441 §3 - f := &forgeFake{refuse: map[string]string{"gitea.gitea_merge_pull_request": "no"}} - p, pub, _, _ := newPerformer(f) - p.merges(context.Background(), "g", 1, mergeKV(t, twoMembers(meshdeliveryclient.MergeMerging, meshdeliveryclient.MergeMerging))) - if len(f.calls) != 1 || len(pub.data) != 1 { - t.Errorf("calls %d messages %d, want the first refusal to stop the pass", len(f.calls), len(pub.data)) - } + t.Run("caught up from the groups", func(t *testing.T) { + groups := gitfixture.Seat(t, seat.Seat, "verbs", "groups answers a group with its members and acts", "verb:groups.answer")[0] + f := &seatFake{answers: map[string]json.RawMessage{"seat:mesh-delivery.groups": groups, + "seat:git.status": answer("status", "status answers")}} + o, pub, kept := newOwing(f) + // The fixture's merge has its outcome recorded; its status is said and unanswered. + if n, err := o.catchUp(context.Background()); err != nil || n != 1 || len(pub.data) != 1 || f.calls[1] != "seat:git.status" { + t.Fatalf("catch-up: %d %v, called %v; want the one unanswered status performed", n, err, f.calls) + } + if n, err := o.catchUp(context.Background()); err != nil || n != 0 || len(f.calls) != 3 { + t.Errorf("catch-up again: %d %v, called %v; want nothing performed that performed holds", n, err, f.calls) + } + if len(kept) != 1 { + t.Errorf("kept %v, want the status", kept) + } + // The seat answers a group it does not know with no groups, never an error (W11.10): nothing is performed and + // nothing fails; a groups the seat leaves unanswered is the catch-up's error. + none := gitfixture.Seat(t, seat.Seat, "verbs", "groups answers a group it does not know with no groups", "verb:groups.answer")[0] + for _, row := range []struct { + answer json.RawMessage + fails bool + }{{none, false}, {nil, true}} { + f := &seatFake{answers: map[string]json.RawMessage{}} + if row.answer != nil { + f.answers["seat:mesh-delivery.groups"] = row.answer + } + o, pub, _ := newOwing(f) + n, err := o.catchUp(context.Background()) + if (err != nil) != row.fails || n != 0 || len(pub.data) != 0 { + t.Errorf("catch-up from %s: %d performed, %v; want none performed, failing %v", row.answer, n, err, row.fails) + } + } + }) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go index 3c0365b8..6ccd2b76 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groupask.go @@ -8,11 +8,13 @@ import ( "github.com/nats-io/nats.go/jetstream" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "mesh-delivery-forge-bridge/internal/groups" ) // endAsked answers one value of the state end-asks (the tool bridge_group_end_again, ADR 0633 §4): a value not yet answered -// says the group's end again from the gate's merges, with the ask's time appended to the message id so the stream's duplicate +// says the group's end again from the seat's groups, with the ask's time appended to the message id so the stream's duplicate // window does not drop it, and is written back by compare-and-set at its revision with whether it was written. A value // answered already (this bridge's own write) is left. func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte) { @@ -24,7 +26,7 @@ func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key strin if a.Done() { return } - a = g.answerEnd(ctx, key, a, time.Now()) + a = g.answerEnd(ctx, a, time.Now()) g.logger.Printf("the end of the group %q said again by %s (%s): %s", a.Group, a.By, a.Why, a.Answer) data, err := json.Marshal(a) if err != nil { @@ -35,27 +37,31 @@ func (g *grouper) endAsked(ctx context.Context, kv jetstream.KeyValue, key strin } } -// readMerges reads one key of the gate's merges state. -func (g *grouper) readMerges(ctx context.Context, key string) ([]byte, uint64, bool, error) { - if g.merges == nil { - return nil, 0, false, errors.New("the bridge does not read the gate's merges (its bucket was not there at the start)") - } - value, rev, found, err := g.merges(ctx, key) - return value, rev, found, err -} - // answerEnd says the group's end again for an ask and returns the ask with its answer. -func (g *grouper) answerEnd(ctx context.Context, key string, a groups.EndAsk, at time.Time) groups.EndAsk { +func (g *grouper) answerEnd(ctx context.Context, a groups.EndAsk, at time.Time) groups.EndAsk { a.Answered = at.UTC() - switch merges, mrev, found, err := g.readMerges(ctx, key); { - case err != nil: - a.Answer = "the gate's merges " + key + " cannot be read: " + err.Error() - case !found: - a.Answer = "the gate holds no merge of the group " + a.Group - case g.end(ctx, key, mrev, merges, groups.AgainStamp(a.At)): + if g.groups == nil { + a.Answer = "the bridge reads no groups" + return a + } + of, err := g.groups(ctx) + if err != nil { + a.Answer = "the groups cannot be read: " + err.Error() + return a + } + m, ok := seat.Group{}, false + for _, c := range of { + if c.ID == a.Group { + m, ok = c, true + } + } + switch { + case !ok || len(m.Members) < 2: + a.Answer = "the seat " + seat.Seat + " holds no group " + a.Group + " of more than one member" + case g.end(ctx, m, groups.AgainStamp(a.At)): a.Written, a.Answer = true, "the end of the group is written again" default: - a.Answer = "the end is not written: a member is not merged or its merged message is not written, or FORGE refused it (the bridge's journal says which)" + a.Answer = "the end is not written: a member's merged message is not written, or FORGE refused it (the bridge's journal says which)" } return a } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go index c840f23a..1a4d65c9 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups.go @@ -2,37 +2,40 @@ package main import ( "context" - "errors" "log" - "slices" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-delivery/sdk/trunk" groupkeys "mesh-delivery-forge-bridge/internal/groups" ) // A delivery group's merges reach the indexer together (novox/hq ADR 0633 §4): the bridge sets the group on each -// member's merged message, from the gate's merges state it reads (ADR 0437 §8), and once every approved member is -// merged and its merged message written on FORGE it says the group's end, forge.group.merged, once. +// member's merged message, from the seat mesh-delivery's verb groups (design 47 *A group's merges are taken in together*; +// it reads no state of the gate's), and once every member's merged message is written on FORGE it says the group's end, +// forge.group.merged, once, after the last of them. // -// What it keeps of it is in its state performed, beside the gate's keys it acted on: `published.` for each -// member's merged message written (the hash is the first 16 hex digits of the sha256 of `#@`), -// and `groups.` once the group's end is written, its value the revision of the gate's merges key it was -// worked out from. A group whose end is not written yet is open (bridge_groups reads them). +// What it keeps of it is in its state performed: `published.` for each member's merged message written, its value +// the merge commit it named (the hash is the first 16 hex digits of the sha256 of `#`), and +// `groups.` once the group's end is written, its value the last member's merge commit. A group whose end is not +// written yet is open (bridge_groups reads them). -// merges reads one key of the gate's merges state. -type mergesRead func(ctx context.Context, key string) (value []byte, rev uint64, found bool, err error) +// groupsRead is every delivery group the seat answers. +type groupsRead func(ctx context.Context) ([]seat.Group, error) + +// strs is performed as string values by key: the grouper's `published.` and `groups.` keys. +type strs interface { + Get(ctx context.Context, key string) (value string, found bool, err error) + Put(ctx context.Context, key, value string) error +} // grouper sets a member's group and says a group's end. type grouper struct { - merges mergesRead + groups groupsRead send func(ctx context.Context, subject string, data []byte, id string) error // writes on FORGE with the message id - done revisions + done strs logger *log.Logger } @@ -44,63 +47,40 @@ func sendVia(pub publisher) func(context.Context, string, []byte, string) error } } -func kvMerges(kv jetstream.KeyValue) mergesRead { - return func(ctx context.Context, key string) ([]byte, uint64, bool, error) { - e, err := kv.Get(ctx, key) - if errors.Is(err, jetstream.ErrKeyNotFound) { - return nil, 0, false, nil - } - if err != nil { - return nil, 0, false, err - } - return e.Value(), e.Revision(), true, nil - } +// groupsVia reads the groups through the seat mesh-delivery's verb groups. +func groupsVia(call caller) groupsRead { + return func(ctx context.Context) ([]seat.Group, error) { return groupkeys.Read(ctx, call) } } -// groupOf is the gate's merge of the group a merged message's branch names, when the message's pull request is one of -// its approved members and the group has more than one: a lone pull request is no group. -func (g *grouper) groupOf(ctx context.Context, f forgemsg.Forge) (seat.MergesByGroupValue, uint64, bool) { - if g == nil || g.merges == nil || f.Branch == "" { - return seat.MergesByGroupValue{}, 0, false +// groupOf is the group a merged message's branch names, when the seat names the message's pull request among its +// members and the group has more than one: a lone pull request is no group. +func (g *grouper) groupOf(ctx context.Context, f forgemsg.Forge) (seat.Group, bool) { + if g == nil || g.groups == nil || f.Branch == "" { + return seat.Group{}, false } - key, err := meshdeliveryclient.MergesKey(f.Branch) + of, err := g.groups(ctx) if err != nil { - return seat.MergesByGroupValue{}, 0, false + g.logger.Printf("%s#%d: the groups cannot be read, so its merged message names no group: %v", f.Repository, f.Number, err) + return seat.Group{}, false } - value, rev, found, err := g.merges(ctx, key) - if err != nil { - g.logger.Printf("%s#%d: the gate's merges.%s cannot be read, so its merged message names no group: %v", f.Repository, f.Number, key, err) - return seat.MergesByGroupValue{}, 0, false - } - if !found { - return seat.MergesByGroupValue{}, 0, false - } - m, err := meshdeliveryclient.DecodeMerge(value) - if err != nil { - g.logger.Printf("%s#%d: the gate's merges.%s is not a merge, so its merged message names no group: %v", f.Repository, f.Number, key, err) - return seat.MergesByGroupValue{}, 0, false - } - if _, ok := m.Approved[meshdeliveryclient.ApprovedKey(f.Repository, f.Number)]; !ok || len(m.Approved) < 2 { - return seat.MergesByGroupValue{}, 0, false - } - return m, rev, true + return groupkeys.Named(of, f.Branch, f.Repository, f.Number) } -// member sets the group and its approved members on a merged message's body (ADR 0633 §4); any other body, and a -// merge of no group, is left as it is. +// member sets the group and its members on a merged message's body (ADR 0633 §4); any other body, and a merge of no +// group, is left as it is. func (g *grouper) member(ctx context.Context, data []byte) ([]byte, forgemsg.Forge, bool) { f, err := forgemsg.DecodeForge(data) if err != nil || f.Kind != forgemsg.ForgeMerged { return data, f, false } - m, _, ok := g.groupOf(ctx, f) + m, ok := g.groupOf(ctx, f) if !ok { return data, f, false } - f.Group, f.Members = m.Group, groupkeys.MembersOf(m) + f.Group, f.Members = m.ID, groupkeys.MembersOf(m) out, err := forgemsg.EncodeForge(f) if err != nil { - g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.Group, err) + g.logger.Printf("%s#%d: its merged message cannot name the group %q, written without: %v", f.Repository, f.Number, m.ID, err) return data, f, false } return out, f, true @@ -108,73 +88,55 @@ func (g *grouper) member(ctx context.Context, data []byte) ([]byte, forgemsg.For // published keeps that a member's merged message is written, and says the group's end when it was the last. func (g *grouper) published(ctx context.Context, f forgemsg.Forge) { - if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number, f.Merge), 1); err != nil { - g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for the next act: %v", f.Repository, f.Number, err) + if err := g.done.Put(ctx, groupkeys.PublishedKey(f.Repository, f.Number), f.Merge); err != nil { + g.logger.Printf("%s#%d: that its merged message is written is not kept, so the group's end waits for bridge_group_end_again: %v", f.Repository, f.Number, err) return } - key, err := meshdeliveryclient.MergesKey(f.Branch) - if err != nil { - return + if m, ok := g.groupOf(ctx, f); ok { + g.end(ctx, m, "") } - value, rev, found, err := g.merges(ctx, key) - if err != nil || !found { - return - } - g.ended(ctx, key, rev, value) } -// ended says the group's end (forge.group.merged) once every approved member of the gate's merge is merged and its -// merged message written, after the last of them; never on a refusal (ADR 0441 §3: a held group merges on from it). -// It is written once per group: `groups.` in performed says it was. again says it is written again by hand +// end says the group's end (forge.group.merged) once every member's merged message is written, after the last of them; +// never on a refusal (ADR 0441 §3: a held group merges on from it), since a refused member's message is never written. It +// is written once per group: `groups.` in performed says it was. again says it is written again by hand // (bridge_group_end_again), with the time appended to the message id, so the duplicate window does not drop it. -func (g *grouper) ended(ctx context.Context, key string, rev uint64, value []byte) { - g.end(ctx, key, rev, value, "") -} - -func (g *grouper) end(ctx context.Context, key string, rev uint64, value []byte, again string) bool { - m, err := meshdeliveryclient.DecodeMerge(value) - if err != nil || len(m.Approved) < 2 { +func (g *grouper) end(ctx context.Context, m seat.Group, again string) bool { + if len(m.Members) < 2 { + return false + } + key, err := groupkeys.Key(m.ID) + if err != nil { + g.logger.Printf("the group %q cannot be kept, so its end is not said: %v", m.ID, err) return false } if again == "" { - if _, seen, err := g.done.Get(ctx, groupkeys.Key(key)); err == nil && seen { + if _, seen, err := g.done.Get(ctx, key); err == nil && seen { return false } } - end := forgemsg.ForgeGroupEnd{Group: m.Group} - for _, member := range groupkeys.MembersOf(m) { - at := slices.IndexFunc(m.Pulls, func(p seat.MergesByGroupValuePullsItem) bool { - return trunk.SameRepository(p.Repository, member.Repository) && int(p.Number) == member.Number - }) - if at < 0 || m.Pulls[at].Commit == "" { - return false // not every member is merged (a merge commit is written only for a merged pull): the group stays gathered - } - if _, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, member.Number, m.Pulls[at].Commit)); err != nil || !ok { + // The members in the group's order, which the gate merges them in (ADR 0441 §5), so the last commit names the end. + end := forgemsg.ForgeGroupEnd{Group: m.ID} + for _, member := range m.Members { + merge, ok, err := g.done.Get(ctx, groupkeys.PublishedKey(member.Repository, int(member.Number))) + if err != nil || !ok || merge == "" { return false // its merged message is not written yet: said when it is (published) } - end.Members = append(end.Members, forgemsg.GroupMerge{Repository: member.Repository, Number: member.Number, Merge: m.Pulls[at].Commit}) + end.Members = append(end.Members, forgemsg.GroupMerge{Repository: member.Repository, Number: int(member.Number), Merge: merge}) } - // The members in the order the gate merged them, so the last commit names the end. - slices.SortStableFunc(end.Members, func(a, b forgemsg.GroupMerge) int { - return slices.IndexFunc(m.Pulls, func(p seat.MergesByGroupValuePullsItem) bool { return p.Commit == a.Merge }) - - slices.IndexFunc(m.Pulls, func(p seat.MergesByGroupValuePullsItem) bool { return p.Commit == b.Merge }) - }) data, err := forgemsg.EncodeForgeGroupEnd(end) if err != nil { - g.logger.Printf("the end of the group %q cannot be written: %v", m.Group, err) + g.logger.Printf("the end of the group %q cannot be written: %v", m.ID, err) return false } - id := groupkeys.EndID(m.Group, end.Members[len(end.Members)-1].Merge, "") - if again != "" { - id += "." + again - } - if err := g.send(ctx, forgeSubject(forgemsg.ForgeGroupEndEvent), data, id); err != nil { - g.logger.Printf("the end of the group %q is not written on FORGE, written at the gate's next write or by bridge_group_end_again: %v", m.Group, err) + last := end.Members[len(end.Members)-1].Merge + if err := g.send(ctx, forgeSubject(forgemsg.ForgeGroupEndEvent), data, groupkeys.EndID(m.ID, last, again)); err != nil { + g.logger.Printf("the end of the group %q is not written on FORGE, written by bridge_group_end_again: %v", m.ID, err) return false } - if err := g.done.Put(ctx, groupkeys.Key(key), rev); err != nil { - g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.Group, err) + if err := g.done.Put(ctx, key, last); err != nil { + g.logger.Printf("the end of the group %q is written and not kept: the stream's duplicate window drops a second one: %v", m.ID, err) } - g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.Group, len(end.Members), end.Members[len(end.Members)-1].Merge) + g.logger.Printf("the end of the group %q is written: %d members merged, the last %s", m.ID, len(end.Members), last) return true } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go index b4100d67..3c24b39e 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/groups_test.go @@ -3,12 +3,12 @@ package main import ( "bytes" "context" + "errors" "log" "strings" "testing" "time" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" @@ -26,32 +26,20 @@ func (p *idPub) send(_ context.Context, subject string, data []byte, id string) return nil } -func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T17, T19 (ADR 0633 §4) +// 1585/T16, T17, T19 (ADR 0633 §4), acceptance 49: a member's merged message names its group from the seat +// mesh-delivery's verb groups, the group's end is said once after the last member's merged message, and said again by hand. +func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { ctx := context.Background() h1, h2 := strings.Repeat("1", 40), strings.Repeat("2", 40) c1, c2 := strings.Repeat("c", 40), strings.Repeat("d", 40) - merge := func(states ...seat.MergesByGroupValuePullsItem) []byte { - v, err := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "feat/take-in", Ask: "ask", Pulls: states, - Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}}) - if err != nil { - t.Fatal(err) - } - return v - } - sdk := func(state meshdeliveryclient.MergeState, commit, words string) seat.MergesByGroupValuePullsItem { - return seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-sdk", Number: 37, Head: h1, State: string(state), Commit: commit, Words: words} - } - dlv := func(state meshdeliveryclient.MergeState, commit string) seat.MergesByGroupValuePullsItem { - return seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-delivery", Number: 9, Head: h2, State: string(state), Commit: commit} - } - gate := map[string][]byte{} + group := seat.Group{ID: "feat/take-in", State: "merging", Members: []seat.Member{ + {Repository: "novox/mesh-sdk", Number: 37, Head: h1, State: "merged"}, {Repository: "novox/mesh-delivery", Number: 9, Head: h2, State: "merging"}}} + var of []seat.Group + var read error var logs bytes.Buffer - pub, done := &idPub{}, memRevisions{} + pub, done := &idPub{}, memStrs{} g := &grouper{send: pub.send, done: done, logger: log.New(&logs, "", 0), - merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { - v, ok := gate[key] - return v, 7, ok, nil - }} + groups: func(context.Context) ([]seat.Group, error) { return of, read }} mergedOf := func(repo string, number int, head, commit string) []byte { data, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repo, Branch: "feat/take-in", Number: number, Head: head, Merge: commit, Root: "r"}) if err != nil { @@ -59,79 +47,64 @@ func TestAGroupsMembersNameItAndItsEndIsSaidOnce(t *testing.T) { // 1585/T16, T1 } return data } - // A lone pull request's merged message carries no group. - if _, f, ok := g.member(ctx, mergedOf("novox/hq", 4, h1, c1)); ok || f.Group != "" { + // A lone pull request's merged message carries no group, and neither does one whose groups cannot be read. + of = []seat.Group{group, {ID: "solo", Members: []seat.Member{{Repository: "novox/hq", Number: 4, Head: h1}}}} + lone, _ := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/hq", Branch: "solo", Number: 4, Head: h1, Merge: c1, Root: "r"}) + if _, f, ok := g.member(ctx, lone); ok || f.Group != "" { t.Errorf("a lone pull request names the group %q", f.Group) } - // The first member: its message names the group and both members; the group is not ended. - gate["feat/take-in"] = merge(sdk(meshdeliveryclient.MergeMerged, c1, ""), dlv(meshdeliveryclient.MergeMerging, "")) + read = errors.New("the seat did not answer") + if _, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1)); ok || f.Group != "" || !strings.Contains(logs.String(), "the seat did not answer") { + t.Errorf("a group read that failed names the group %q: %s", f.Group, logs.String()) + } + read = nil + // The first member: its message names the group and both members in the group's order; the group is not ended. data, f, ok := g.member(ctx, mergedOf("novox/mesh-sdk", 37, h1, c1)) - if back, err := forgemsg.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 { + if back, err := forgemsg.DecodeForge(data); !ok || err != nil || back.Group != "feat/take-in" || len(back.Members) != 2 || back.Members[0].Number != 37 { t.Fatalf("a member's message: %+v %v", back, err) } g.published(ctx, f) - // A refusal of the other member is no end (ADR 0441 §3: the group merges on from it later). - gate["feat/take-in"] = merge(sdk(meshdeliveryclient.MergeMerged, c1, ""), seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-delivery", Number: 9, Head: h2, State: string(meshdeliveryclient.MergeRefused), Words: "405"}) - g.ended(ctx, "feat/take-in", 8, gate["feat/take-in"]) - // Merged on at the gate before its merged message is written: still no end; then its message ends the group. - gate["feat/take-in"] = merge(sdk(meshdeliveryclient.MergeMerged, c1, ""), dlv(meshdeliveryclient.MergeMerged, c2)) - g.ended(ctx, "feat/take-in", 9, gate["feat/take-in"]) - if len(pub.subjects) != 0 { - t.Fatalf("an end before every member's merged message is written: %v", pub.subjects) + if len(pub.subjects) != 0 || done[groupkeys.PublishedKey("novox/mesh-sdk", 37)] != c1 { + t.Fatalf("after the first member: ends %v, performed %v", pub.subjects, done) } + // The last member's merged message ends the group; a second merged message of it says no second end. _, f, _ = g.member(ctx, mergedOf("novox/mesh-delivery", 9, h2, c2)) g.published(ctx, f) - g.ended(ctx, "feat/take-in", 10, gate["feat/take-in"]) // a later write of the gate says no second end + g.published(ctx, f) if len(pub.subjects) != 1 || pub.subjects[0] != "mesh.stream.mesh-delivery-forge-bridge.event.forge.group.merged" || pub.ids[0] != "group-end.feat/take-in."+c2 { t.Fatalf("ends %v ids %v, want one forge.group.merged with id group-end.feat/take-in.%s", pub.subjects, pub.ids, c2) } end, err := forgemsg.DecodeForgeGroupEnd(pub.data[0]) if err != nil || len(end.Members) != 2 || end.Members[0].Merge != c1 || end.Members[1].Merge != c2 { - t.Errorf("the end = %+v %v, want both members in their merge order", end, err) + t.Errorf("the end = %+v %v, want both members in the group's order", end, err) } - if rev, ok := done[groupkeys.Key("feat/take-in")]; !ok || rev != 7 { - t.Errorf("performed %v: want groups.feat/take-in kept", done) - } - // Again by hand: written once more, its id not the first's. - if !g.end(ctx, "feat/take-in", 10, gate["feat/take-in"], "20261104T2300") || pub.ids[1] == pub.ids[0] { - t.Errorf("the end again by hand: ids %v", pub.ids) - } - // 1585/T19: an ask to say a group's end again is answered by the bridge from the gate's merges, with the ask's time on - // the message id; not written while a member is not merged, with the reason; not for a group the gate does not hold. - pull := func(repo string, n int, head string, state meshdeliveryclient.MergeState, commit string) seat.MergesByGroupValuePullsItem { - return seat.MergesByGroupValuePullsItem{Repository: repo, Number: int64(n), Head: head, State: string(state), Commit: commit} - } - askMerge := func(second meshdeliveryclient.MergeState, commit2 string) []byte { - v, err := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "feat/take-in", Ask: "ask", Approved: map[string]string{"novox/mesh-sdk@37": h1, "novox/mesh-delivery@9": h2}, - Pulls: []seat.MergesByGroupValuePullsItem{pull("novox/mesh-sdk", 37, h1, meshdeliveryclient.MergeMerged, c1), pull("novox/mesh-delivery", 9, h2, second, commit2)}}) - if err != nil { - t.Fatal(err) - } - return v + if key, _ := groupkeys.Key("feat/take-in"); done[key] != c2 { + t.Errorf("performed %v: want groups.feat~take-in kept with the last merge", done) } + // 1585/T19: an ask to say a group's end again is answered from the seat's groups, with the ask's time on the message id; + // not written while a member's merged message is not written, with the reason; not for a group the seat does not hold, + // nor when the groups cannot be read. ask := groupkeys.EndAsk{V: 1, Group: "feat/take-in", By: "operator@shanks", Why: "the indexer never took it in", At: time.Date(2026, 11, 4, 23, 0, 0, 123, time.UTC)} for _, tc := range []struct { name string - gate map[string][]byte + of []seat.Group + read error published bool written bool answer string }{ - {"merged and published: written again", map[string][]byte{"feat/take-in": askMerge(meshdeliveryclient.MergeMerged, c2)}, true, true, "written again"}, - {"a member's message not written", map[string][]byte{"feat/take-in": askMerge(meshdeliveryclient.MergeMerged, c2)}, false, false, "not written"}, - {"a member not merged", map[string][]byte{"feat/take-in": askMerge(meshdeliveryclient.MergeMerging, "")}, true, false, "not written"}, - {"no merge at the gate", map[string][]byte{}, true, false, "holds no merge"}, + {"every message written: written again", []seat.Group{group}, nil, true, true, "written again"}, + {"a member's message not written", []seat.Group{group}, nil, false, false, "not written"}, + {"no such group at the seat", []seat.Group{{ID: "other", Members: group.Members}}, nil, true, false, "holds no group"}, + {"the groups cannot be read", nil, errors.New("no answer"), true, false, "cannot be read"}, } { - pub, done := &idPub{}, memRevisions{} + pub, done := &idPub{}, memStrs{} if tc.published { - done[groupkeys.PublishedKey("novox/mesh-sdk", 37, c1)], done[groupkeys.PublishedKey("novox/mesh-delivery", 9, c2)] = 1, 1 + done[groupkeys.PublishedKey("novox/mesh-sdk", 37)], done[groupkeys.PublishedKey("novox/mesh-delivery", 9)] = c1, c2 } g := &grouper{send: pub.send, done: done, logger: log.New(&bytes.Buffer{}, "", 0), - merges: func(_ context.Context, key string) ([]byte, uint64, bool, error) { - v, ok := tc.gate[key] - return v, 4, ok, nil - }} - got := g.answerEnd(ctx, "feat/take-in", ask, ask.At.Add(time.Second)) + groups: func(context.Context) ([]seat.Group, error) { return tc.of, tc.read }} + got := g.answerEnd(ctx, ask, ask.At.Add(time.Second)) if got.Written != tc.written || !strings.Contains(got.Answer, tc.answer) || !got.Done() || len(pub.ids) != map[bool]int{true: 1}[tc.written] { t.Errorf("%s: %+v, ids %v", tc.name, got, pub.ids) } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go index 2c54ba13..a33ce668 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -20,28 +21,27 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-forge-bridge/internal/gitfixture" + groupkeys "mesh-delivery-forge-bridge/internal/groups" ) -// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, -// the controller's user, and the bridge's own principal . with the permissions the controller composes for -// it, and no others, so a permission the bridge lacks fails a test here and not on the mesh (issue 1279). +// busConf is the shape of the composed accounts file (the grants part's buscompose): one account with JetStream, the +// controller's user, and the bridge's own principal . with the permissions composed for its manifest, and +// no others, so a permission the bridge lacks fails a test here and not on the mesh (issue 1279). // -// The gate's buckets mesh-delivery-gate_merges and mesh-delivery-gate_owed are read and watched (the manifest's reads), the -// bucket performed is the bridge's own (its state), and the five forge tools of the gate's acts are invoked (issue 1341, 1342). -// -// The bridge neither reads the checker's owed nor hears its stream (ADR 0521, R4e finding R2-1): the forge module writes -// mesh/check from it. The stream checks is made here so a test can say the checker wrote a status. -// -// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal -// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls, -// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, performed, inputs and end-asks (ADR 0633 §5's hand end), and the stream forge; `**` is written `>` on the wire (modstreams.go -// WirePattern). The consumer's durable is _ on EVENTS (nats.go consumerDurable). The controller's own -// user keeps all rights, as it has. +// From module.json: the seat git's events heard through its durable on EVENTS; what the gate and the checker owe heard +// through its durable on each seat's own stream owed (SEAT__owed); the seat git's verbs, the seat mesh-delivery's +// groups and the seat mesh-hand-acts' hand-act called on mesh.seat..call.. (or its plain fallback); +// its own states heads, performed, inputs, end-asks and owed-asks; its stream forge. It reads no state of the gate's or +// the checker's (design 47, acceptance 49 and 53). `**` is written `>` on the wire. The controller's own user keeps all +// rights, as it has. const busConf = ` listen: 127.0.0.1:-1 jetstream { store_dir: %q } @@ -53,38 +53,23 @@ accounts { { user: "n1.mesh-delivery-forge-bridge", password: "pw" permissions { publish { allow: [ - "$JS.ACK.EVENTS.n1_mesh-delivery-forge-bridge.>" "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_heads.>" "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_heads.>" - "$JS.API.CONSUMER.INFO.EVENTS.n1_mesh-delivery-forge-bridge" - "$JS.API.CONSUMER.MSG.NEXT.EVENTS.n1_mesh-delivery-forge-bridge" "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_heads.>" "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_heads" "$JS.FC.KV_mesh-delivery-forge-bridge_heads.>" "$KV.mesh-delivery-forge-bridge_heads.>" - "mesh.mod.gitea.call.gitea_commit_trees.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_commit_trees.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_open_pulls.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_open_pulls.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_commit_trees" - "mesh.mod.gitea.tool.gitea_commit_trees.*" - "mesh.mod.gitea.call.gitea_get_file.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_get_file.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_get_file" - "mesh.mod.gitea.tool.gitea_get_file.*" - "mesh.mod.gitea.tool.gitea_open_pulls" - "mesh.mod.gitea.tool.gitea_open_pulls.*" "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_inputs.>" "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_inputs.>" "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_inputs.>" "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_performed" - "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_inputs" "$JS.FC.KV_mesh-delivery-forge-bridge_performed.>" - "$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>" "$KV.mesh-delivery-forge-bridge_performed.>" + "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_inputs.>" + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_inputs.>" + "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_inputs.>" + "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_inputs" + "$JS.FC.KV_mesh-delivery-forge-bridge_inputs.>" "$KV.mesh-delivery-forge-bridge_inputs.>" "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_end-asks.>" "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_end-asks.>" @@ -92,45 +77,51 @@ accounts { "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_end-asks" "$JS.FC.KV_mesh-delivery-forge-bridge_end-asks.>" "$KV.mesh-delivery-forge-bridge_end-asks.>" - "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_merges.>" - "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_merges.>" - "$JS.API.DIRECT.GET.KV_mesh-delivery-gate_merges.>" - "$JS.API.STREAM.INFO.KV_mesh-delivery-gate_merges" - "$JS.FC.KV_mesh-delivery-gate_merges.>" - "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-gate_owed.>" - "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-gate_owed.>" - "$JS.API.DIRECT.GET.KV_mesh-delivery-gate_owed.>" - "$JS.API.STREAM.INFO.KV_mesh-delivery-gate_owed" - "$JS.FC.KV_mesh-delivery-gate_owed.>" - "mesh.mod.gitea.call.gitea_merge_pull_request.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_merge_pull_request.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_merge_pull_request" - "mesh.mod.gitea.tool.gitea_merge_pull_request.*" - "mesh.mod.gitea.call.gitea_commit_status.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_commit_status.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_commit_status" - "mesh.mod.gitea.tool.gitea_commit_status.*" - "mesh.mod.gitea.call.gitea_delivery_view.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_delivery_view.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_delivery_view" - "mesh.mod.gitea.tool.gitea_delivery_view.*" - "mesh.mod.gitea.call.gitea_note_append.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_note_append.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_note_append" - "mesh.mod.gitea.tool.gitea_note_append.*" - "mesh.mod.gitea.call.gitea_add_comment.*.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.call.gitea_add_comment.n1~mesh-delivery-forge-bridge" - "mesh.mod.gitea.tool.gitea_add_comment" - "mesh.mod.gitea.tool.gitea_add_comment.*" + "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_owed-asks.>" + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_owed-asks.>" + "$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_owed-asks.>" + "$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_owed-asks" + "$JS.FC.KV_mesh-delivery-forge-bridge_owed-asks.>" + "$KV.mesh-delivery-forge-bridge_owed-asks.>" + "$JS.ACK.EVENTS.n1_mesh-delivery-forge-bridge.>" + "$JS.API.CONSUMER.INFO.EVENTS.n1_mesh-delivery-forge-bridge" + "$JS.API.CONSUMER.MSG.NEXT.EVENTS.n1_mesh-delivery-forge-bridge" + "$JS.ACK.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge.>" + "$JS.API.CONSUMER.INFO.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge" + "$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-delivery_owed.n1_mesh-delivery-forge-bridge" + "$JS.ACK.SEAT_mesh-delivery-checker_owed.n1_mesh-delivery-forge-bridge.>" + "$JS.API.CONSUMER.INFO.SEAT_mesh-delivery-checker_owed.n1_mesh-delivery-forge-bridge" + "$JS.API.CONSUMER.MSG.NEXT.SEAT_mesh-delivery-checker_owed.n1_mesh-delivery-forge-bridge" + "mesh.seat.git.call.status.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.status" + "mesh.seat.git.call.comment.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.comment" + "mesh.seat.git.call.view.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.view" + "mesh.seat.git.call.note.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.note" + "mesh.seat.git.call.merge-pull.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.merge-pull" + "mesh.seat.git.call.open-pulls.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.open-pulls" + "mesh.seat.git.call.trees.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.trees" + "mesh.seat.git.call.file.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.file" + "mesh.seat.git.call.pull.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.pull" + "mesh.seat.git.call.tags.n1~mesh-delivery-forge-bridge" + "mesh.seat.git.tool.tags" + "mesh.seat.mesh-delivery.call.groups.n1~mesh-delivery-forge-bridge" + "mesh.seat.mesh-delivery.tool.groups" + "mesh.seat.mesh-hand-acts.call.hand-act.n1~mesh-delivery-forge-bridge" + "mesh.seat.mesh-hand-acts.tool.hand-act" "mesh.stream.mesh-delivery-forge-bridge.event.forge.>" + "mesh.stream.mesh-delivery-forge-bridge.event.owed.ended" ] } subscribe { allow: [ "_INBOX.n1.mesh-delivery-forge-bridge.>" "mesh.assignment.n1.mesh-delivery-forge-bridge" - "mesh.mod.gitea.event.forge.delete" - "mesh.mod.gitea.event.pull.closed" - "mesh.mod.gitea.event.pull.merged" - "mesh.mod.gitea.event.pull.updated" "mesh.mod.mesh-delivery-forge-bridge.call.>" "mesh.mod.mesh-delivery-forge-bridge.tool.>" ] } @@ -186,26 +177,32 @@ type bed struct { ctx context.Context done <-chan error reply func(string) - // files sets what the stand-in forge answers to gitea_get_file from then on; it is asked with the tool's arguments. - files func(func(owner, repo, path, ref string) (string, bool)) + // files sets what the stand-in seat git answers to file from then on; it is asked with the verb's input. + files func(func(repository, path, ref string) (string, bool)) + // answer sets the stand-in's answer to one verb of a seat (`.`) from then on. + answer func(verb string, answer string) + // calls are the verbs the stand-in was called with, in order, as `. `. + calls func() []string + // restart stops the bridge, runs between, and starts it again on the same bus. + restart func(between func()) } -// getFileReply is the reply envelope for gitea_get_file as mesh-catalog modules/gitea tools/index.ts and client.ts getFile -// answer it ({file: {path, ref, sha, size, content}}), and for a path the forge does not have the error client.ts request -// writes: `Gitea API : 404 `. -func getFileReply(path, ref, content string, found bool) []byte { +// answerOf is a verb's answer in the reply envelope of mesh-sdk go/process/call.go ({result, error, node}). +func answerOf(answer string) []byte { return []byte(`{"result":` + answer + `,"node":"n0"}`) } + +// fileAnswer is the seat git's answer to file: the content, or absent. +func fileAnswer(path, ref, content string, found bool) string { if !found { - b, _ := json.Marshal(map[string]any{"error": "Gitea API /repos/novox/mesh-delivery/contents/" + path + "?ref=" + ref + `: 404 {"message":"GetContentsOrList","url":"https://forge.example/api/swagger"}`}) - return b + return `{"absent":true}` } - b, _ := json.Marshal(map[string]any{"result": map[string]any{"file": map[string]any{"path": path, "ref": ref, "sha": "0123456789abcdef0123456789abcdef01234567", - "size": len(content), "content": content}}, "node": "n0"}) - return b + b, _ := json.Marshal(git.FileAnswer{Path: path, Ref: ref, Content: content, Size: int64(len(content))}) + return string(b) } -// startBed is startBridge with a step between the streams and the consumer: prepare writes what the streams hold before -// the bridge's consumer is made, as the controller makes it, from now (ADR 0443 §1), and reply sets the answer of the -// forge's gitea_open_pulls from then on. +// startBed is startBridge with a step between the streams and the consumers: prepare writes what the streams hold before +// the bridge's consumers are made, as the grants part makes them, from now (ADR 0443 §1), and reply sets the answer of the +// seat git's open-pulls from then on. A stand-in holds the seats the bridge calls: git (every verb, trees r0 by default, +// a module.json of {} at every path), mesh-delivery's groups (none by default) and mesh-hand-acts' hand-act. func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, js jetstream.JetStream)) *bed { srv := startBus(t) ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw")) @@ -219,70 +216,74 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"}}); err != nil { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.>"}}); err != nil { t.Fatal(err) } - // The checker's stream checks, which the bridge no longer hears (ADR 0521): a test says on it that the checker wrote a status. - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil { - t.Fatal(err) + // Each seat's own stream owed, as its contract keeps it (Stream, SEAT__owed), as the grants part makes it: no + // subjects of its own, sourced from EVENTS filtered by the event's subject. + for name, subjects := range map[string]string{seat.StreamOwed: seat.SubjectOwedAll, checker.StreamOwed: checker.SubjectOwedAll} { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: name, MaxAge: 7 * 24 * time.Hour, + Sources: []*jetstream.StreamSource{{Name: "EVENTS", FilterSubject: subjects}}}); err != nil { + t.Fatal(err) + } } - // The bridge's state bucket, made by the controller from the manifest's state entry before the process starts. - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: headsBucket, History: 1}); err != nil { - t.Fatal(err) - } - // Its other state bucket, and the gate's buckets the bridge reads, made by the controller from the gate's manifest - // (the gate's merges and owed with their histories). - for name, history := range map[string]uint8{performedBucket: 1, inputsBucket: 1, mergesBucket: 10, owedBucket: 5} { - if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil { + // The bridge's states, made from the manifest's state entries and its requires before the process starts. + for _, name := range []string{headsBucket, performedBucket, inputsBucket, groupkeys.EndAsksBucket, groupkeys.OwedAsksBucket} { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 1}); err != nil { t.Fatal(err) } } if prepare != nil { prepare(ctx, js) } + acct := process.Account{Node: "n1", Module: "mesh-delivery-forge-bridge"} if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{ - Durable: "n1_mesh-delivery-forge-bridge", AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy, - FilterSubjects: []string{"mesh.mod.gitea.event.pull.updated", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.closed", "mesh.mod.gitea.event.forge.delete"}, + Durable: acct.EventConsumerName(), AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverNewPolicy, + FilterSubjects: []string{git.SubjectPullUpdated, git.SubjectPullMerged, git.SubjectPullClosed, git.SubjectForgeDelete}, }); err != nil { t.Fatal(err) } + for name, subjects := range map[string]string{seat.StreamOwed: seat.SubjectOwedAll, checker.StreamOwed: checker.SubjectOwedAll} { + if _, err := js.CreateOrUpdateConsumer(ctx, name, acct.SeatEventConsumerConfig([]string{subjects}, false)); err != nil { + t.Fatal(err) + } + } - // A stand-in answers the forge's gitea_open_pulls, which the bridge asks at its start and after a reconnect, on its tool - // subject (mesh-control internal/broker/nats.go invokedSubjects) in the reply envelope of mesh-sdk go/process/call.go. + // The stand-in seats, answering on the caller-named subject and its plain fallback. var mu sync.Mutex - current := openReply - open, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_open_pulls", func(m *nats.Msg) { - mu.Lock() - defer mu.Unlock() - _ = m.Respond([]byte(current)) - }) - if err != nil { - t.Fatal(err) + answers := map[string]string{ + "git.open-pulls": openReply, "git.trees": `{"root":"r0","trees":{"":"t0","modules/a":"t1"}}`, + "mesh-delivery.groups": `{"groups":[]}`, "mesh-hand-acts.hand-act": `{"recorded":true}`, } - t.Cleanup(func() { _ = open.Unsubscribe() }) - if err := ctl.Flush(); err != nil { - t.Fatal(err) + for _, verb := range []string{"status", "comment", "view", "note", "merge-pull"} { + answers["git."+verb] = string(gitfixture.Given(t, verb+" answers", "verb:"+verb+".answer")) } - - // A stand-in answers the forge's gitea_get_file: by default a module.json of {} at every path and no other file. - var fmu sync.Mutex - filesOf := func(_, _, path, _ string) (string, bool) { return "{}", strings.HasSuffix(path, "module.json") } - getFile, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_get_file", func(m *nats.Msg) { - var a struct{ Owner, Repo, Path, Ref string } - _ = json.Unmarshal(m.Data, &a) - fmu.Lock() - f := filesOf - fmu.Unlock() - content, found := f(a.Owner, a.Repo, a.Path, a.Ref) - _ = m.Respond(getFileReply(a.Path, a.Ref, content, found)) - }) - if err != nil { - t.Fatal(err) + filesOf := func(_, path, _ string) (string, bool) { return "{}", strings.HasSuffix(path, "module.json") } + var called []string + for _, subject := range []string{"mesh.seat.*.call.*.*", "mesh.seat.*.tool.*"} { + sub, err := ctl.Subscribe(subject, func(m *nats.Msg) { + parts := strings.Split(m.Subject, ".") + verb := parts[2] + "." + parts[4] + mu.Lock() + called = append(called, verb+" "+string(m.Data)) + answer, files := answers[verb], filesOf + mu.Unlock() + if verb == "git.file" { + var in git.FileInput + _ = json.Unmarshal(m.Data, &in) + content, found := files(in.Repository, in.Path, in.Ref) + answer = fileAnswer(in.Path, in.Ref, content, found) + } + _ = m.Respond(answerOf(answer)) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) } - t.Cleanup(func() { _ = getFile.Unsubscribe() }) if err := ctl.Flush(); err != nil { t.Fatal(err) } @@ -293,24 +294,37 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context, t.Fatal(err) } env := map[string]string{"MESH_BROKER_FILE": broker} - runCtx, stop := context.WithCancel(ctx) done := make(chan error, 1) - go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() - t.Cleanup(stop) - return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { mu.Lock(); current = s; mu.Unlock() }, - files: func(f func(owner, repo, path, ref string) (string, bool)) { fmu.Lock(); filesOf = f; fmu.Unlock() }} + var stop context.CancelFunc + start := func() { + var runCtx context.Context + runCtx, stop = context.WithCancel(ctx) + go func() { done <- run(runCtx, func(k string) string { return env[k] }, io.Discard) }() + } + start() + t.Cleanup(func() { stop() }) + set := func(verb, answer string) { mu.Lock(); answers[verb] = answer; mu.Unlock() } + return &bed{srv: srv, ctl: ctl, js: js, ctx: ctx, done: done, reply: func(s string) { set("git.open-pulls", s) }, + files: func(f func(repository, path, ref string) (string, bool)) { mu.Lock(); filesOf = f; mu.Unlock() }, + answer: set, + calls: func() []string { mu.Lock(); defer mu.Unlock(); return append([]string(nil), called...) }, + restart: func(between func()) { + stop() + <-done + between() + start() + }} } // TestBridgeWritesPullMessages starts the program the way the node does (its entry, run, with MESH_BROKER_FILE). func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2 - ctl, js, ctx, _ := startBridge(t, `{"result":{"pulls":[]}}`) + ctl, js, ctx, _ := startBridge(t, `{"pulls":[]}`) defer branchDeletedCase(t, ctl.ConnectedUrl(), js, ctx) - // gitea.pull.updated as the forge module publishes it: the body of trunk.ts:233-250 on mesh.mod.gitea.event.pull.updated. - ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "t", "body": "b", "base": "main", - "head": "feat/x", "head_sha": head, "clone_url": "u", "html_url": "h", "paths": []string{"modules/a/x.go"}, "paths_truncated": false, - "removed": []string{}, "module_dirs": []string{"modules/a"}, "module_dirs_said": true}) - if _, err := js.Publish(ctx, "mesh.mod.gitea.event.pull.updated", ev); err != nil { + // pull.updated as the seat git says it (its contract's PullUpdatedEvent). + ev, _ := json.Marshal(git.PullUpdatedEvent{Repository: "novox/mesh-delivery", Number: 12, Title: "t", Base: "main", Branch: "feat/x", Head: head, + Change: &git.Change{Paths: []string{"modules/a/x.go"}, ModuleDirs: []string{"modules/a"}}}) + if _, err := js.Publish(ctx, git.SubjectPullUpdated, ev); err != nil { t.Fatal(err) } @@ -357,13 +371,13 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2 t.Errorf("the event is not acknowledged: %+v", ci) } -// branchDeletedCase is DLV-FB1's case of the bridge's whole-module test (design 51): the forge module's forge.delete of -// a branch is a branch-deleted message on FORGE, and a forge.delete from another account than the forge module's is -// refused by the bus. +// branchDeletedCase is DLV-FB1's case of the bridge's whole-module test (design 51): the seat git's forge.delete of a +// branch is a branch-deleted message on FORGE, and a forge.delete from another account than the seat's holder is refused +// by the bus. func branchDeletedCase(t *testing.T, srv string, js jetstream.JetStream, ctx context.Context) { t.Helper() - ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-sdk", "ref": "feat/x", "ref_type": "branch", "sha": head}) - if _, err := js.Publish(ctx, "mesh.mod.gitea.event.forge.delete", ev); err != nil { + ev, _ := json.Marshal(git.ForgeDeleteEvent{Repository: "novox/mesh-sdk", Ref: "feat/x", RefType: "branch"}) + if _, err := js.Publish(ctx, git.SubjectForgeDelete, ev); err != nil { t.Fatal(err) } forge, err := js.Stream(ctx, "FORGE") @@ -388,21 +402,26 @@ func branchDeletedCase(t *testing.T, srv string, js jetstream.JetStream, ctx con ojs, _ := jetstream.New(other) octx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() - if _, err := ojs.Publish(octx, "mesh.mod.gitea.event.forge.delete", ev); err == nil { - t.Error("a forge.delete from the bridge's account was taken by the bus: only the forge module's account may say it") + if _, err := ojs.Publish(octx, git.SubjectForgeDelete, ev); err == nil { + t.Error("a forge.delete from the bridge's account was taken by the bus: only the seat git's holder may say it") } } const merge = "89abcdef0123456789abcdef0123456789abcdef" -// TestBridgeWritesMergedMessage has a stand-in answer the forge's gitea_commit_trees on its tool subject -// (mesh-control internal/broker/nats.go invokedSubjects: mesh.mod..tool.), in the reply envelope of -// mesh-sdk go/process/call.go ({result, error, node}). +// openPull is one entry of the seat git's open-pulls answer. +func openPull(repo string, number int, branch, head string) string { + b, _ := json.Marshal(git.OpenPull{Pull: git.Pull{Repository: "novox/" + repo, Number: int64(number), Base: "main", Branch: branch, Head: head, State: "open"}, + Checks: map[string]git.Status{}, Required: []string{}}) + return string(b) +} + +// TestBridgeWritesMergedMessage: the stand-in seat git answers trees and file on the caller-named subjects. func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 - bed := startBed(t, `{"result":{"pulls":[]}}`, nil) - ctl, js, ctx := bed.ctl, bed.js, bed.ctx + bed := startBed(t, `{"pulls":[]}`, nil) + js, ctx := bed.js, bed.ctx // The stand-in forge's files are two real files of this repository (this module's module.json and go.sum, read from the - // working tree), in gitea_get_file's real answer shape: the root has a manifest only, modules/a a manifest and go.sum. + // working tree), in the seat git's file answer: the root has a manifest only, modules/a a manifest and go.sum. manifest, err := os.ReadFile("../../module.json") if err != nil { t.Fatal(err) @@ -411,7 +430,7 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 if err != nil { t.Fatal(err) } - bed.files(func(_, _, path, ref string) (string, bool) { + bed.files(func(_, path, ref string) (string, bool) { if ref != merge { t.Errorf("the forge was asked for %s at %s, want the merge commit %s", path, ref, merge) } @@ -423,25 +442,10 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 } return "", false }) - var asked chan []byte = make(chan []byte, 4) - sub, err := ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) { - asked <- m.Data - _ = m.Respond([]byte(`{"result":{"root":"r0","trees":{"":"t0","modules/a":"t1"}},"node":"n0"}`)) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - if err := ctl.Flush(); err != nil { - t.Fatal(err) - } - - // gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 publishes it. - ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "title": "t", "body": "b", - "head_sha": head, "head": "feat/x", "base": "main", "merge_commit_sha": merge, "merged_at": "2026-10-19T12:00:00Z", - "clone_url": "u", "html_url": "h", "paths": []string{"modules/a/x.go"}, "paths_truncated": false, "removed": []string{}, - "module_dirs": []string{"modules/a"}, "module_dirs_said": true}) - if _, err := js.Publish(ctx, "mesh.mod.gitea.event.pull.merged", ev); err != nil { + // pull.merged as the seat git says it (its contract's PullMergedEvent). + ev, _ := json.Marshal(git.PullMergedEvent{Repository: "novox/mesh-delivery", Number: 12, Title: "t", Head: head, Branch: "feat/x", Base: "main", + MergeCommit: merge, Change: &git.Change{Paths: []string{"modules/a/x.go"}, ModuleDirs: []string{"modules/a"}}}) + if _, err := js.Publish(ctx, git.SubjectPullMerged, ev); err != nil { t.Fatal(err) } @@ -475,16 +479,15 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 if !forgemsg.SameForge(f, want) { t.Errorf("message = %+v, want %+v", f, want) } - select { - case body := <-asked: - if string(body) != `{"owner":"novox","repo":"mesh-delivery","sha":"`+merge+`"}` { - t.Errorf("the tool was asked %s", body) + var trees []string + for _, c := range bed.calls() { + if strings.HasPrefix(c, "git.trees ") { + trees = append(trees, c) } - default: - t.Error("the stand-in was not asked") } - if len(asked) != 0 { - t.Error("the tool was asked more than once") + in, _ := json.Marshal(git.TreesInput{Repository: "novox/mesh-delivery", Commit: merge}) + if len(trees) != 1 || trees[0] != "git.trees "+string(in) { + t.Errorf("the seat git was asked for trees %v, want once at the merge commit", trees) } cons, err := js.Consumer(ctx, "EVENTS", "n1_mesh-delivery-forge-bridge") if err != nil { @@ -503,10 +506,10 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4, 1448/AC5 // TestBridgeSaysTheOpenPullsAtStart: the forge holds two open pull requests when the bridge starts; one message each is on // FORGE, and the open message naming both after them (ADR 0443 §5). func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1 - // The entries are gitea_open_pulls's (mesh-catalog modules/gitea/client.ts:369-381, head the branch). - _, js, ctx, _ := startBridge(t, `{"result":{"pulls":[ - {"owner":"novox","repo":"mesh-delivery","number":12,"base":"main","head":"feat/x","head_sha":"`+head+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}, - {"owner":"novox","repo":"mesh-catalog","number":300,"base":"main","head":"feat/x","head_sha":"`+strings.Repeat("a", 40)+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}]},"node":"n0"}`) + // The entries are the seat git's open-pulls answer (its contract's OpenPull). + _, js, ctx, _ := startBridge(t, `{"pulls":[ + `+openPull("mesh-delivery", 12, "feat/x", head)+`, + `+openPull("mesh-catalog", 300, "feat/x", strings.Repeat("a", 40))+`]}`) forge, err := js.Stream(ctx, "FORGE") if err != nil { t.Fatal(err) @@ -533,10 +536,10 @@ func TestBridgeSaysTheOpenPullsAtStart(t *testing.T) { // 1237/AC1 // TestBridgeExitsWhenTheOpenPullsCannotBeRead: the process manager starts it again. func TestBridgeExitsWhenTheOpenPullsCannotBeRead(t *testing.T) { // 1237/AC3 - _, _, _, done := startBridge(t, `{"error":"the forge said 502"}`) + _, _, _, done := startBridge(t, `{"error":"the forge said 502","error-kind":"failed","pulls":[]}`) select { case err := <-done: - if err == nil || !strings.Contains(err.Error(), "gitea.gitea_open_pulls") || !strings.Contains(err.Error(), "the forge said 502") { + if err == nil || !strings.Contains(err.Error(), "open-pulls") || !strings.Contains(err.Error(), "the forge said 502") { t.Errorf("run returned %v", err) } case <-time.After(10 * time.Second): @@ -555,14 +558,12 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes return "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox." + repo + ".pull" } openSubject := "mesh.stream.mesh-delivery-forge-bridge.event.forge.open" - b := startBed(t, `{"result":{"pulls":[ - {"owner":"novox","repo":"mesh-delivery","number":12,"base":"main","head":"feat/x","head_sha":"`+head+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}, - {"owner":"novox","repo":"mesh-catalog","number":300,"base":"main","head":"feat/x","head_sha":"`+h2+`","html_url":"h","required":[],"statuses":[],"checks":{},"labels":[]}]},"node":"n0"}`, + b := startBed(t, `{"pulls":[`+openPull("mesh-delivery", 12, "feat/x", head)+`,`+openPull("mesh-catalog", 300, "feat/x", h2)+`]}`, func(ctx context.Context, js jetstream.JetStream) { // Older events: a pull request long closed and one long updated, from before the bridge's consumer. - for subject, ev := range map[string]map[string]any{ - "mesh.mod.gitea.event.pull.updated": {"owner": "novox", "repo": "mesh-old", "number": 5, "head": "feat/old", "head_sha": h2}, - "mesh.mod.gitea.event.pull.closed": {"owner": "novox", "repo": "mesh-older", "number": 6, "head": "feat/older", "head_sha": h2}, + for subject, ev := range map[string]any{ + git.SubjectPullUpdated: git.PullUpdatedEvent{Repository: "novox/mesh-old", Number: 5, Base: "main", Branch: "feat/old", Head: h2}, + git.SubjectPullClosed: git.PullClosedEvent{Repository: "novox/mesh-older", Number: 6, Base: "main", Branch: "feat/older", Head: h2}, } { raw, _ := json.Marshal(ev) if _, err := js.Publish(ctx, subject, raw); err != nil { @@ -651,171 +652,186 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes } } -// TestBridgeMergesAGroupOneMemberAtATime: the gate (played by the test) writes merges.g with member A handed and the -// Approve's heads of A and B; a stand-in forge merges A and announces it; the gate then hands B; both end merged on FORGE -// (1342/AC3, ADR 0437 §8's check). -func TestBridgeMergesAGroupOneMemberAtATime(t *testing.T) { // 1342/AC3 - b := startBed(t, `{"result":{"pulls":[]}}`, nil) - const headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - const mergeA, mergeB = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222" - merges := make(chan string, 4) - // The stand-in for gitea_merge_pull_request answers in the reply envelope of mesh-sdk go/process/call.go and publishes - // gitea.pull.merged as mesh-catalog modules/gitea/index.ts:186-207 does, with the merge commit of the member merged. - sub, err := b.ctl.Subscribe("mesh.mod.gitea.tool.gitea_merge_pull_request", func(m *nats.Msg) { - var a struct { - Repo string `json:"repo"` - Head string `json:"head"` - } - _ = json.Unmarshal(m.Data, &a) - merges <- string(m.Data) - commit, number := mergeA, 12 - if a.Repo == "mesh-sdk" { - commit, number = mergeB, 4 - } - ev, _ := json.Marshal(map[string]any{"owner": "novox", "repo": a.Repo, "number": number, "head_sha": a.Head, "head": "feat/x", - "base": "main", "merge_commit_sha": commit, "module_dirs": []string{"modules/a"}, "module_dirs_said": true}) - _, _ = b.js.Publish(b.ctx, "mesh.mod.gitea.event.pull.merged", ev) - _ = m.Respond([]byte(`{"result":{"merged":true},"node":"n0"}`)) - }) +// heard is every message published under the bridge's stream subjects from now, by subject. +func heard(t *testing.T, b *bed) <-chan *nats.Msg { + t.Helper() + ch := make(chan *nats.Msg, 64) + sub, err := b.ctl.ChanSubscribe("mesh.stream.mesh-delivery-forge-bridge.event.>", ch) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) - trees, err := b.ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_trees", func(m *nats.Msg) { - _ = m.Respond([]byte(`{"result":{"root":"r0","trees":{"":"t0","modules/a":"t1"}},"node":"n0"}`)) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = trees.Unsubscribe() }) if err := b.ctl.Flush(); err != nil { t.Fatal(err) } + return ch +} - kv, err := b.js.KeyValue(b.ctx, mergesBucket) - if err != nil { - t.Fatal(err) - } - approved := map[string]string{meshdeliveryclient.ApprovedKey("novox/mesh-delivery", 12): head, meshdeliveryclient.ApprovedKey("novox/mesh-sdk", 4): headB} - pullA := seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-delivery", Number: 12, Head: head, State: string(meshdeliveryclient.MergeMerging)} - pullB := seat.MergesByGroupValuePullsItem{Repository: "novox/mesh-sdk", Number: 4, Head: headB, State: string(meshdeliveryclient.MergeMerging)} - put := func(pulls ...seat.MergesByGroupValuePullsItem) { - v, err := meshdeliveryclient.EncodeMerge(seat.MergesByGroupValue{Group: "g", Ask: "merge.g.0123456789ab", Approved: approved, Pulls: pulls}) - if err != nil { - t.Fatal(err) - } - if _, err := kv.Put(b.ctx, "g", v); err != nil { - t.Fatal(err) - } - } - next := func(repo string) { +// next is the next message heard on subject within a bound; the others are passed by. +func next(t *testing.T, ch <-chan *nats.Msg, subject string) []byte { + t.Helper() + bound := time.After(20 * time.Second) + for { select { - case got := <-merges: - if !strings.Contains(got, `"repo":"`+repo+`"`) { - t.Errorf("the forge was asked %s, want %s", got, repo) + case m := <-ch: + if m.Subject == subject { + return m.Data } - for _, want := range []string{"novox/mesh-delivery@" + head, "novox/mesh-sdk@" + headB} { - if !strings.Contains(got, want) { - t.Errorf("the merge %s does not carry the approved head %s", got, want) - } - } - case <-time.After(10 * time.Second): - t.Fatalf("%s was not merged within 10 s", repo) + case <-bound: + t.Fatalf("nothing on %s within 20 s", subject) + return nil } } - put(pullA) - next("mesh-delivery") - // The gate hands the next member only after the bridge reported the one before merged (ADR 0441 §5). - for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline) && len(forgeMerged(t, b)) < 1; time.Sleep(50 * time.Millisecond) { - } - pullA.State, pullA.Commit = string(meshdeliveryclient.MergeMerged), mergeA - put(pullA, pullB) - next("mesh-sdk") - var got []string - for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { - if got = forgeMerged(t, b); len(got) == 2 { - break - } - } - if len(got) != 2 { - t.Fatalf("FORGE holds merged messages for %v, want both members", got) - } - select { - case extra := <-merges: - t.Errorf("a third merge was asked: %s", extra) - default: - } } -// forgeMerged lists the repositories FORGE holds a merged message for, sorted. -func forgeMerged(t *testing.T, b *bed) []string { +// refusedRead says the bridge's account cannot read the bucket: a key-value read of another module's state is refused. +func refusedRead(t *testing.T, b *bed, bucket string) { t.Helper() - forge, err := b.js.Stream(b.ctx, "FORGE") + if _, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: bucket}); err != nil { + t.Fatal(err) + } + nc, err := nats.Connect(b.srv.ClientURL(), nats.UserInfo("n1.mesh-delivery-forge-bridge", "pw")) if err != nil { t.Fatal(err) } - var out []string - for _, repo := range []string{"mesh-delivery", "mesh-sdk"} { - raw, err := forge.GetLastMsgForSubject(b.ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox."+repo+".merged") - if err != nil { - continue - } - if f, err := forgemsg.DecodeForge(raw.Data); err == nil && f.Kind == forgemsg.ForgeMerged { - out = append(out, f.Repository) + defer nc.Close() + js, _ := jetstream.New(nc) + ctx, cancel := context.WithTimeout(b.ctx, 2*time.Second) + defer cancel() + if kv, err := js.KeyValue(ctx, bucket); err == nil { + if _, err := kv.Get(ctx, "any"); err == nil || errors.Is(err, jetstream.ErrKeyNotFound) { + t.Errorf("the bridge's account reads %s", bucket) } } - return out +} + +// Acceptance 49: the forge bridge sets the group on each member's merged message and says the group's end once, after the +// last member's, from the seat mesh-delivery's verb groups; its account reads no state of the gate's. +func TestBridgeSaysAGroupFromTheSeatsGroups(t *testing.T) { // 1585, ADR 0633 §4 + b := startBed(t, `{"pulls":[]}`, nil) + headB := strings.Repeat("b", 40) + mergeB := strings.Repeat("e", 40) + groups, _ := json.Marshal(seat.GroupsAnswer{Groups: []seat.Group{{ID: "feat/x", State: "merging", Members: []seat.Member{ + {Repository: "novox/mesh-delivery", Number: 12, Head: head, State: "merged"}, {Repository: "novox/mesh-sdk", Number: 4, Head: headB, State: "merging"}}}}}) + b.answer("mesh-delivery.groups", string(groups)) + ch := heard(t, b) + for _, m := range []git.PullMergedEvent{ + {Repository: "novox/mesh-delivery", Number: 12, Head: head, Branch: "feat/x", Base: "main", MergeCommit: merge, Change: &git.Change{Paths: []string{"modules/a/x.go"}, ModuleDirs: []string{"modules/a"}}}, + {Repository: "novox/mesh-sdk", Number: 4, Head: headB, Branch: "feat/x", Base: "main", MergeCommit: mergeB, Change: &git.Change{Paths: []string{"go/x.go"}}}, + } { + ev, _ := json.Marshal(m) + if _, err := b.js.Publish(b.ctx, git.SubjectPullMerged, ev); err != nil { + t.Fatal(err) + } + repo := strings.TrimPrefix(m.Repository, "novox/") + f, err := forgemsg.DecodeForge(next(t, ch, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox."+repo+".merged")) + if err != nil || f.Group != "feat/x" || len(f.Members) != 2 || f.Members[1].Number != 4 { + t.Fatalf("%s's merged message: %+v %v, want the group feat/x with both members", repo, f, err) + } + } + end, err := forgemsg.DecodeForgeGroupEnd(next(t, ch, forgeSubject(forgemsg.ForgeGroupEndEvent))) + if err != nil || end.Group != "feat/x" || len(end.Members) != 2 || end.Members[0].Merge != merge || end.Members[1].Merge != mergeB { + t.Errorf("the group's end: %+v %v", end, err) + } + refusedRead(t, b, "mesh-delivery-gate_merges") +} + +// Acceptance 53 (design 47 *What the gate owes the forge*): an owed act said on a seat's own stream owed is performed through +// the seat git and its outcome said as owed.ended; said twice it is performed once; one it cannot read is said undecodable; +// one said while the bridge is stopped is performed once after its restart; bridge_owed_again's ask is answered by the +// process; and its account reads neither the gate's nor the checker's state. +func TestBridgePerformsWhatIsOwedOnce(t *testing.T) { + b := startBed(t, `{"pulls":[]}`, nil) + ch := heard(t, b) + ended := func() forgemsg.OwedEnded { + t.Helper() + e, err := forgemsg.DecodeOwedEnded(next(t, ch, forgeSubject(forgemsg.OwedEndedEvent))) + if err != nil { + t.Fatal(err) + } + return e + } + count := func(verb string) int { + n := 0 + for _, c := range b.calls() { + if strings.HasPrefix(c, verb+" ") { + n++ + } + } + return n + } + gateOwed := gitfixture.Seat(t, seat.Seat, "events", "owed is said, a merge", "event:owed")[0] + subject := seat.SubjectOwed("novox~mesh-sdk", "178") + for range 2 { + if _, err := b.js.Publish(b.ctx, subject, gateOwed); err != nil { + t.Fatal(err) + } + } + if e := ended(); e.Outcome != forgemsg.OwedPerformed || e.Of != seat.Seat { + t.Errorf("the gate's merge: %+v, want performed", e) + } + if _, err := b.js.Publish(b.ctx, subject, []byte("{")); err != nil { + t.Fatal(err) + } + if e := ended(); e.Outcome != forgemsg.OwedUndecodable { + t.Errorf("an act it cannot read: %+v, want undecodable", e) + } + if n := count("git.merge-pull"); n != 1 { + t.Errorf("an act said twice was performed %d times", n) + } + // Said while the bridge is stopped: performed once after its restart. + checkerOwed := gitfixture.Seat(t, checker.Seat, "events", "owed is said", "event:owed")[0] + b.restart(func() { + if _, err := b.js.Publish(b.ctx, checker.SubjectOwed("novox~mesh-sdk", "178"), checkerOwed); err != nil { + t.Fatal(err) + } + }) + if e := ended(); e.Of != checker.Seat || e.Outcome != forgemsg.OwedPerformed { + t.Errorf("the checker's status after the restart: %+v, want performed", e) + } + if n := count("git.status"); n != 1 { + t.Errorf("the checker's status was performed %d times, want once", n) + } + // bridge_owed_again asks through the state owed-asks; the process performs the act again and answers there. + var g struct { + ID string `json:"id"` + } + _ = json.Unmarshal(gateOwed, &g) + key, value, err := groupkeys.NewOwedAsk(groupkeys.OwedAgain, seat.Seat, g.ID, "a test", "lead@n1", time.Now()) + if err != nil { + t.Fatal(err) + } + asks, err := b.js.KeyValue(b.ctx, groupkeys.OwedAsksBucket) + if err != nil { + t.Fatal(err) + } + w, err := asks.Watch(b.ctx, key) + if err != nil { + t.Fatal(err) + } + defer w.Stop() + if _, err := asks.Put(b.ctx, key, value); err != nil { + t.Fatal(err) + } + bound := time.After(20 * time.Second) + for answered := false; !answered; { + select { + case e := <-w.Updates(): + if e == nil { + continue + } + if a, err := groupkeys.DecodeOwedAsk(e.Value()); err == nil && a.Done() { + answered = true + if a.Outcome != forgemsg.OwedPerformed || count("git.merge-pull") != 2 { + t.Errorf("bridge_owed_again: %+v, merge-pull asked %d times; want performed again", a, count("git.merge-pull")) + } + } + case <-bound: + t.Fatal("bridge_owed_again was not answered within 20 s") + } + } + refusedRead(t, b, "mesh-delivery-gate_owed") + refusedRead(t, b, "mesh-delivery-checker_owed") } func sha256Sum(b []byte) []byte { s := sha256.Sum256(b); return s[:] } - -// TestBridgePerformsNoMeshCheckAct: the checker owes mesh/check on a head and says so on its stream; the bridge calls no -// tool with mesh/check and publishes nothing of its own on EVENTS for it. The forge module writes mesh/check from the -// checker's owed state itself (ADR 0521; R4e round 2 finding R2-1). -func TestBridgePerformsNoMeshCheckAct(t *testing.T) { // R2-1, ADR 0521 - b := startBed(t, `{"result":{"pulls":[]}}`, nil) - calls := make(chan string, 8) - // Every forge call is watched, and none answered here: the bed's stand-ins answer the ones the bridge asks. - sub, err := b.ctl.Subscribe("mesh.mod.gitea.>", func(m *nats.Msg) { - if strings.Contains(string(m.Data), "mesh/check") { - calls <- m.Subject + " " + string(m.Data) - } - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - if err := b.ctl.Flush(); err != nil { - t.Fatal(err) - } - kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: "mesh-delivery-checker_owed", History: 5}) - if err != nil { - t.Fatal(err) - } - key, err := meshdeliveryclient.OwedKey("novox/mesh-delivery", 12, head, "status=mesh/check") - if err != nil { - t.Fatal(err) - } - v, err := meshdeliveryclient.EncodeOwed(seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: meshdeliveryclient.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.0123456789ab"}) - if err != nil { - t.Fatal(err) - } - if _, err := kv.Put(b.ctx, key, v); err != nil { - t.Fatal(err) - } - if _, err := b.js.Publish(b.ctx, "mesh.stream.mesh-delivery-checker.event.owed.written", []byte(`{"key":"`+key+`"}`)); err != nil { - t.Fatal(err) - } - select { - case c := <-calls: - t.Errorf("a forge tool was called with mesh/check: %s", c) - case <-time.After(2 * time.Second): - } - events, err := b.js.Stream(b.ctx, "EVENTS") - if err != nil { - t.Fatal(err) - } - if _, err := events.GetLastMsgForSubject(b.ctx, "mesh.mod.mesh-delivery-forge-bridge.event.>"); err == nil { - t.Error("the bridge published an event of its own for the checker's mesh/check") - } -} diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go index 19ad49af..f91af302 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go @@ -22,6 +22,8 @@ import ( "github.com/nats-io/nats.go/jetstream" git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "mesh-delivery-forge-bridge/internal/bridge" @@ -146,39 +148,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // Everything written on FORGE, an event's and a sync's, keeps the head in the bucket. rec := &recorder{pub: js, heads: heads, logger: logger} said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) } - // What the gate asks of the forge (ADR 0437 §8): its owed acts and its merges, each performed once per revision. - performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "perform an act of the gate only once") + performedKV, err := process.EssentialBucket(ctx, js, "bridge", performedBucket, "keep what it performed of what is owed to the forge") if err != nil { return err } - perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger} - // A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the gate's merges. - groups := &grouper{send: sendVia(js), done: kvRevisions{performedKV}, logger: logger} - watching := make(chan error, 3) - for _, w := range []struct { - bucket string - react func(ctx context.Context, key string, rev uint64, value []byte) - }{{owedBucket, perf.owed}, {mergesBucket, func(ctx context.Context, key string, rev uint64, value []byte) { - perf.merges(ctx, key, rev, value) - groups.ended(ctx, key, rev, value) - }}} { - // The gate's buckets come with the gate and its grant; the bridge does not wait for them (no timer), and is started - // again by its delivery when the gate arrives. A bucket refused or not answered is the same (issue 1945). - kv := process.OptionalBucket(ctx, js, w.bucket, "nothing of it is performed until the bridge starts again", logger.Printf) - if kv == nil { - continue - } - if w.bucket == mergesBucket { - groups.merges = kvMerges(kv) - } - go func() { - watching <- watch(ctx, kv, func(key string, rev uint64, value []byte) { - hctx, cancel := context.WithTimeout(ctx, syncDeadline) - defer cancel() - w.react(hctx, key, rev, value) - }) - }() - } + // What the gate and the checker owe the forge, performed through the seat git (design 47 *What the gate owes the forge*). + owe := &owing{call: call, pub: js, performed: kvPerformed{performedKV}, now: time.Now, logger: logger} + // A delivery group's member names its group, and its end is said once (ADR 0633 §4), from the seat mesh-delivery's groups. + groups := &grouper{groups: groupsVia(call), send: sendVia(js), done: kvHeads{performedKV}, logger: logger} // A person says a group's end again with bridge_group_end_again, which writes the state end-asks (ADR 0633 §4): watched, and // answered there; a state not there at this start is answered from the next start. if kv := process.OptionalBucket(ctx, js, groupkeys.EndAsksBucket, "bridge_group_end_again is not answered until the bridge starts again", logger.Printf); kv != nil { @@ -192,6 +169,43 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := said(ctx); err != nil { return err } + // What the gate and the checker say they owe, each on its seat's own stream owed through this module's durable there + // (made from its consumes). A stream not there yet is said, and the bridge goes on: its other work does not wait on the + // gate, and what was said meanwhile is caught up from the groups at the next start. + for _, owner := range []struct{ of, kept string }{{seat.Seat, seat.StreamOwed}, {checker.Seat, checker.StreamOwed}} { + cc, err := cred.HearSeatEvents(ctx, js, owner.kept, func(m jetstream.Msg) { + hctx, cancel := context.WithTimeout(ctx, syncDeadline) + defer cancel() + owe.hear(hctx, owner.of, m) + }, logger.Printf) + if err != nil { + logger.Printf("nothing %s owes the forge is performed until the bridge starts again: %v", owner.of, err) + continue + } + defer cc.Stop() + } + // A person performs an act again, drops one or reads the consumers' backlog with the tools bridge_owed_again, + // bridge_owed_drop and bridge_owed, which write the state owed-asks: watched, and answered there. + if kv := process.OptionalBucket(ctx, js, groupkeys.OwedAsksBucket, "bridge_owed_again, bridge_owed_drop and bridge_owed's backlog are not answered until the bridge starts again", logger.Printf); kv != nil { + backlog := owedBacklog(js, cred, time.Now) + go func() { + err := watch(ctx, kv, func(key string, rev uint64, value []byte) { + hctx, cancel := context.WithTimeout(ctx, syncDeadline) + defer cancel() + owe.asked(hctx, kv, key, rev, value, backlog) + }) + if err != nil && ctx.Err() == nil { + logger.Printf("the owed asks are no longer watched, so they are not answered until the bridge starts again: %v", err) + } + }() + } + // What was said while the bridge was away and is still unanswered, read from the seat (design 47 *The bridge catches up + // from the seat*). The gate comes after the bridge, so a seat that does not answer yet is said, and the bridge goes on. + if n, err := owe.catchUp(ctx); err != nil { + logger.Printf("nothing owed is caught up from the groups at this start, so an act said while the bridge was away waits for its saying again: %v", err) + } else if n > 0 { + logger.Printf("%d acts owed to the forge, said while the bridge was away, performed from the groups", n) + } go onReconnect(ctx, again, said, logger) // What a merge read of each module directory, by its tree (issue 1126): a merge reads only the directories it changed. inputs := newKept(process.OptionalBucket(ctx, js, inputsBucket, "a merge's module directories are kept in memory only, so the first merge after a start reads every one", logger.Printf), logger) @@ -213,10 +227,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return fmt.Errorf("%w: the bridge hears no forge event without it, so it stops; where the bus refused it or did not answer, this module's consumes (made by the controller with its grant) have not reached the bus yet", err) } defer hub.Stop() - select { - case <-ctx.Done(): - return nil - case err := <-watching: - return fmt.Errorf("watching the gate's keys: %w", err) - } + <-ctx.Done() + return nil } diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names.go index 4e59a705..0cc0db7d 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names.go @@ -2,15 +2,9 @@ package main import groupkeys "mesh-delivery-forge-bridge/internal/groups" -// The holders and buckets the forge bridge names, taken from its own manifest (module.json: its name, its `reads`, its +// The holders and buckets the forge bridge names, taken from its own manifest (module.json: its name and its // states); a unit row checks them against it (TestTheNamesAreTheManifests). No contract names a holder // (novox/hq ADR 0625 §4, ADR 0634 §1). const ( forgeBridgeModule = groupkeys.Module - gateModule = "mesh-delivery-gate" - - // mergesBucket and owedBucket are the gate's states merges and owed, read by the bridge (reads: mesh-delivery-gate.merges, - // mesh-delivery-gate.owed). - mergesBucket = gateModule + "_merges" - owedBucket = gateModule + "_owed" ) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go index 95746226..f2e9e83f 100644 --- a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/names_test.go @@ -7,8 +7,8 @@ import ( "testing" ) -// theNamesAreTheManifests (a subtest of TestAnOwedStatusIsSetOnceForARevision): the holders and buckets the bridge names -// are its manifest's (its module, its `reads`, its states), since no contract names a holder (novox/hq ADR 0634 §1). +// theNamesAreTheManifests (a subtest of TestWhatIsOwedIsPerformedSaidAndKept): the holders and buckets the bridge names +// are its manifest's (its module, its `reads`, its states and its kept data in `requires`), since no contract names a holder (novox/hq ADR 0634 §1). func theNamesAreTheManifests(t *testing.T) { raw, err := os.ReadFile("../../module.json") if err != nil { @@ -20,6 +20,7 @@ func theNamesAreTheManifests(t *testing.T) { State []struct { Name string `json:"name"` } `json:"state"` + Requires []json.RawMessage `json:"requires"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) @@ -27,15 +28,22 @@ func theNamesAreTheManifests(t *testing.T) { if m.Module != forgeBridgeModule { t.Errorf("the manifest's module is %q, the bridge names itself %q", m.Module, forgeBridgeModule) } - for _, read := range []string{gateModule + ".merges", gateModule + ".owed"} { - if !slices.Contains(m.Reads, read) { - t.Errorf("the bridge reads %s, its manifest's reads are %v", read, m.Reads) - } + if len(m.Reads) != 0 { + t.Errorf("the bridge reads no other module's state (acceptance 49), its manifest's reads are %v", m.Reads) } var own []string for _, s := range m.State { own = append(own, forgeBridgeModule+"_"+s.Name) } + for _, r := range m.Requires { + var kept struct { + Name string `json:"name"` + Template string `json:"template"` + } + if json.Unmarshal(r, &kept) == nil && kept.Template == "state" { + own = append(own, forgeBridgeModule+"_"+kept.Name) + } + } for _, b := range []string{headsBucket, performedBucket, inputsBucket} { if !slices.Contains(own, b) { t.Errorf("the bridge keeps %s, its manifest's states are %v", b, own) diff --git a/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go new file mode 100644 index 00000000..7066e2bc --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/owed.go @@ -0,0 +1,342 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log" + "strconv" + "strings" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + + "git.novox.be/novox/mesh-sdk/go/process" + + "mesh-delivery-forge-bridge/internal/bridge" + "mesh-delivery-forge-bridge/internal/groups" +) + +// performedBucket is the module's state performed (manifest `requires`, the template state): one key per owed act it +// performed, the act's outcome and when, with the owed event as heard (novox/hq design 47 *What the gate owes the +// forge*). It is a saving and a read, never what keeps an act from happening twice: every act is safe to repeat at the +// forge. The group ends' bookkeeping (`published.`, `groups.`, groups.go) is in it too, under the same 7 days. +const performedBucket = forgeBridgeModule + "_performed" + +// performedKey is the key of one act in performed: the seat that said it and its id, the id in unpadded base64url, since +// an id may hold what a key may not (`#`, `/`). +func performedKey(of, id string) string { + return of + "." + base64.RawURLEncoding.EncodeToString([]byte(id)) +} + +// Performed is one value of the state performed: the outcome said as owed.ended, and the owed event it answers, so +// bridge_owed_again performs it again from here. +type Performed struct { + Ended forgemsg.OwedEnded `json:"ended"` + Owed json.RawMessage `json:"owed,omitempty"` +} + +// performedStore is the state performed as the owing reads and writes it. +type performedStore interface { + Get(ctx context.Context, key string) (value []byte, found bool, err error) + Put(ctx context.Context, key string, value []byte) error +} + +type kvPerformed struct{ kv jetstream.KeyValue } + +func (s kvPerformed) Get(ctx context.Context, key string) ([]byte, bool, error) { + e, err := s.kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return e.Value(), true, nil +} + +func (s kvPerformed) Put(ctx context.Context, key string, value []byte) error { + _, err := s.kv.Put(ctx, key, value) + return err +} + +// owing performs what the gate and the checker owe the forge (design 47 *What the gate owes the forge*): each owed act +// through the seat git's verb under its id, its outcome said as owed.ended on the stream forge and then kept in +// performed. It holds no timer. +type owing struct { + call caller + pub publisher + performed performedStore + now func() time.Time + logger *log.Logger +} + +// readerOf is the reading of the seat of's owed event into the seat git's verb. +func readerOf(of string) (func([]byte) (bridge.OwedCall, error), error) { + switch of { + case seat.Seat: + return bridge.GateOwed, nil + case checker.Seat: + return bridge.CheckerOwed, nil + } + return nil, fmt.Errorf("the seat %q says no owed act the bridge performs", of) +} + +// heard performs one owed event the seat of said. Its error is only an outcome that could not be said on the stream +// forge: the event is then heard again, and the act, safe to repeat, performed again. +func (o *owing) heard(ctx context.Context, of string, data []byte) (forgemsg.OwedEnded, error) { + read, err := readerOf(of) + if err != nil { + return o.end(ctx, forgemsg.OwedEnded{Of: of, Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, data) + } + c, err := read(data) + if err != nil { + return o.end(ctx, forgemsg.OwedEnded{Of: of, ID: c.ID, + Outcome: forgemsg.OwedUndecodable, Words: err.Error()}, data) + } + if o.held(ctx, c.Of, c.ID) { + // said twice, or said again while its outcome was on its way: performed once, the outcome said once + return forgemsg.OwedEnded{Of: c.Of, ID: c.ID}, nil + } + if words, done := o.switchedDone(ctx, c.Of, c.ID); done { + // performed by the old bridge before the switch: said performed, never done twice + return o.end(ctx, forgemsg.OwedEnded{Of: c.Of, ID: c.ID, + Outcome: forgemsg.OwedPerformed, Words: words}, data) + } + return o.perform(ctx, c, data) +} + +// switchedDone says whether a switched act (design 47 *The switch carries what is owed*: the gate's act of a key of its +// old state owed or merges, its id meshdeliveryclient.SwitchedActID) was performed by the old bridge: its own bookkeeping +// in performed, `.`, holds that key at the act's revision or a later one. Any other id, and bookkeeping that +// cannot be read, is not: the act is performed, which changes nothing at the forge. +func (o *owing) switchedDone(ctx context.Context, of, id string) (string, bool) { + state, key, rev, ok := meshdeliveryclient.ParseSwitchedActID(id) + if !ok || of != seat.Seat { + return "", false + } + kept, found, err := o.performed.Get(ctx, state+"."+key) + if err != nil || !found { + return "", false + } + at, err := strconv.ParseUint(strings.TrimSpace(string(kept)), 10, 64) + if err != nil || at < rev { + return "", false + } + return fmt.Sprintf("performed by the bridge before the switch: its %s.%s at revision %d", state, key, at), true +} + +// hear is one message of a seat's stream owed: performed and acknowledged, or, when its outcome could not be said, +// left to come again (design 47: the bridge stopped between an act and its owed.ended performs it again). +func (o *owing) hear(ctx context.Context, of string, m delivered) { + if _, err := o.heard(ctx, of, m.Data()); err != nil { + o.logger.Printf("%s, so the act is heard again: %v", m.Subject(), err) + _ = m.NakWithDelay(retryDelay) + return + } + _ = m.Ack() +} + +// perform calls the seat git's verb of one act and says what it came to. +func (o *owing) perform(ctx context.Context, c bridge.OwedCall, data []byte) (forgemsg.OwedEnded, error) { + answer, callErr := o.call(ctx, "seat:"+git.Seat+"."+c.Verb, c.Input) + outcome, words := bridge.Outcome(answer, callErr) + if outcome != forgemsg.OwedPerformed { + o.logger.Printf("the owed %s %s of %s#%d (said by %s) is %s: %s", c.Act, c.ID, c.Repository, c.Number, c.Of, outcome, words) + } + return o.end(ctx, forgemsg.OwedEnded{Of: c.Of, ID: c.ID, + Outcome: outcome, Words: words}, data) +} + +// end says an act's outcome as owed.ended on the stream forge, then keeps it in performed. An outcome not kept is +// logged: the act may be performed again, which changes nothing at the forge. +func (o *owing) end(ctx context.Context, e forgemsg.OwedEnded, owed []byte) (forgemsg.OwedEnded, error) { + e.At = o.now().UTC() + data, err := forgemsg.EncodeOwedEnded(e) + if err != nil { + return e, fmt.Errorf("the outcome of the owed %s cannot be said: %w", e.ID, err) + } + if _, err := o.pub.Publish(ctx, forgeSubject(forgemsg.OwedEndedEvent), data); err != nil { + return e, fmt.Errorf("the outcome %s of the owed %s could not be said on the stream forge: %w", e.Outcome, e.ID, err) + } + if e.ID == "" { + return e, nil // an undecodable act with no id is said, and has no key to keep + } + if !json.Valid(owed) { + owed = nil + } + v, err := json.Marshal(Performed{Ended: e, Owed: owed}) + if err == nil { + err = o.performed.Put(ctx, performedKey(e.Of, e.ID), v) + } + if err != nil { + o.logger.Printf("the outcome %s of the owed %s is said but could not be kept in performed: %v", e.Outcome, e.ID, err) + } + return e, nil +} + +// held says whether performed holds an outcome of the act already. One that cannot be read is not held: the act is +// performed again, which changes nothing at the forge. +func (o *owing) held(ctx context.Context, of, id string) bool { + _, found, err := o.performed.Get(ctx, performedKey(of, id)) + if err != nil { + o.logger.Printf("performed of the owed %s cannot be read, so it is performed again: %v", id, err) + return false + } + return found +} + +// catchUp reads the seat mesh-delivery's groups and performs every act said there with no recorded outcome that +// performed does not hold either (design 47 *The bridge catches up from the seat*). It answers how many it performed. +func (o *owing) catchUp(ctx context.Context) (int, error) { + of, err := groups.Read(ctx, o.call) + if err != nil { + return 0, err + } + n := 0 + for _, g := range of { + for _, act := range g.Acts { + if act.Outcome != "" || o.held(ctx, seat.Seat, act.Owed.ID) { + continue // answered, or performed and its outcome on its way to the gate + } + data, err := json.Marshal(act.Owed) + if err != nil { + return n, err + } + if _, err := o.heard(ctx, seat.Seat, data); err != nil { + return n, err + } + n++ + } + } + return n, nil +} + +// owedOf is the owed event of an act to perform again: as performed keeps it, else, for the gate's, as its groups say it. +func (o *owing) owedOf(ctx context.Context, of, id string) ([]byte, error) { + v, found, err := o.performed.Get(ctx, performedKey(of, id)) + if err != nil { + return nil, fmt.Errorf("performed cannot be read: %w", err) + } + if found { + var p Performed + if err := json.Unmarshal(v, &p); err == nil && len(p.Owed) > 0 { + return p.Owed, nil + } + } + if of != seat.Seat { + return nil, fmt.Errorf("the bridge holds no owed act %s of %s: the checker says its unanswered acts again at its start", id, of) + } + raw, err := o.call(ctx, "seat:"+seat.Seat+"."+seat.VerbGroups, seat.GroupsInput{All: seat.GroupsInputAllTrue}) + if err != nil { + return nil, fmt.Errorf("the seat %s's groups could not be read: %w", seat.Seat, err) + } + var a seat.GroupsAnswer + if err := json.Unmarshal(raw, &a); err != nil || a.Error != "" { + return nil, fmt.Errorf("the seat %s's groups do not answer: %v %s", seat.Seat, err, a.Error) + } + for _, g := range a.Groups { + for _, act := range g.Acts { + if act.Owed.ID == id { + return json.Marshal(act.Owed) + } + } + } + return nil, fmt.Errorf("neither the bridge nor the seat %s's groups hold an owed act %s", seat.Seat, id) +} + +// answer answers one owed ask: an act performed again now, whatever performed holds of it; an act dropped, said as +// owed.ended with the outcome dropped and the ask's why (the tool wrote the hand-act log first); or the consumers' backlog. +func (o *owing) answer(ctx context.Context, a groups.OwedAsk, backlog func(context.Context) []groups.Backlog) groups.OwedAsk { + a.Answered = o.now().UTC() + switch a.Kind { + case groups.OwedBacklog: + a.Backlog, a.Answer = backlog(ctx), "the backlog of each seat's stream owed" + return a + } + data, err := o.owedOf(ctx, a.Of, a.ID) + if err != nil { + a.Answer = err.Error() + return a + } + var e forgemsg.OwedEnded + if a.Kind == groups.OwedDrop { + e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedDropped, Words: a.Why + " (dropped by " + a.By + ")"}, data) + } else if read, rerr := readerOf(a.Of); rerr != nil { + err = rerr + } else if c, rerr := read(data); rerr != nil { + e, err = o.end(ctx, forgemsg.OwedEnded{Of: a.Of, ID: a.ID, Outcome: forgemsg.OwedUndecodable, Words: rerr.Error()}, data) + } else { + e, err = o.perform(ctx, c, data) + } + if err != nil { + a.Answer = err.Error() + return a + } + a.Outcome, a.Answer = e.Outcome, "said as owed.ended: "+e.Outcome + if e.Words != "" { + a.Answer += ": " + e.Words + } + return a +} + +// asked answers one value of the state owed-asks not yet answered, written back by compare-and-set at its revision; an +// answered one (this bridge's own write) is left. +func (o *owing) asked(ctx context.Context, kv jetstream.KeyValue, key string, rev uint64, value []byte, backlog func(context.Context) []groups.Backlog) { + a, err := groups.DecodeOwedAsk(value) + if err != nil { + o.logger.Printf("%s.%s holds what is not an owed ask, left: %v", groups.OwedAsksState, key, err) + return + } + if a.Done() { + return + } + a = o.answer(ctx, a, backlog) + o.logger.Printf("the owed ask %s of %s %s by %s (%s): %s", a.Kind, a.Of, a.ID, a.By, a.Why, a.Answer) + data, err := json.Marshal(a) + if err != nil { + return + } + if _, err := kv.Update(ctx, key, data, rev); err != nil && !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) { + o.logger.Printf("%s.%s: the answer could not be written: %v", groups.OwedAsksState, key, err) + } +} + +// owedBacklog reads the bridge's durable on each seat's stream owed: its acknowledged floor, what waits and what is held, +// and the time of the first event after the floor. +func owedBacklog(js jetstream.JetStream, cred process.Account, now func() time.Time) func(context.Context) []groups.Backlog { + return func(ctx context.Context) []groups.Backlog { + var out []groups.Backlog + for _, kept := range []string{seat.StreamOwed, checker.StreamOwed} { + row := groups.Backlog{Stream: kept, Consumer: cred.EventConsumerName()} + c, err := js.Consumer(ctx, row.Stream, row.Consumer) + var info *jetstream.ConsumerInfo + if err == nil { + info, err = c.Info(ctx) + } + if err != nil { + row.Error = err.Error() + out = append(out, row) + continue + } + row.LastApplied, row.Waiting, row.Held = info.AckFloor.Stream, info.NumPending, uint64(info.NumAckPending) + if row.Waiting+row.Held > 0 { + if s, err := js.Stream(ctx, row.Stream); err == nil { + if m, err := s.GetMsg(ctx, info.AckFloor.Stream+1); err == nil { + row.Oldest, row.OldestSecs = m.Time.UTC(), int64(now().Sub(m.Time).Seconds()) + } + } + } + out = append(out, row) + } + return out + } +} diff --git a/modules/mesh-delivery-forge-bridge/forgemsg/forge.go b/modules/mesh-delivery-forge-bridge/forgemsg/forge.go index 822de445..03568714 100644 --- a/modules/mesh-delivery-forge-bridge/forgemsg/forge.go +++ b/modules/mesh-delivery-forge-bridge/forgemsg/forge.go @@ -156,7 +156,7 @@ type Forge struct { PredefinedRead bool `json:"predefined-read,omitempty"` // Group is the delivery group a merged message's pull request was merged in, and Members every approved member of // that group with its approved head, this one included (novox/hq ADR 0633 §4): the forge bridge sets both from the - // gate's merges state, and the indexer gathers the group's merges until the group's end (ForgeGroupEnd). A merge + // seat mesh-delivery's verb groups, and the indexer gathers the group's merges until the group's end (ForgeGroupEnd). A merge // with no group carries neither, and an older bridge sends neither; an older reader ignores both. Group string `json:"group,omitempty"` Members []meshdeliveryclient.Head `json:"members,omitempty"` diff --git a/modules/mesh-delivery-forge-bridge/go.mod b/modules/mesh-delivery-forge-bridge/go.mod index 9a5a262e..e7af2fcd 100644 --- a/modules/mesh-delivery-forge-bridge/go.mod +++ b/modules/mesh-delivery-forge-bridge/go.mod @@ -3,8 +3,10 @@ module mesh-delivery-forge-bridge go 1.26.0 require ( + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go b/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go deleted file mode 100644 index 65e58eff..00000000 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/acts.go +++ /dev/null @@ -1,160 +0,0 @@ -package bridge - -import ( - "fmt" - "slices" - "strings" - - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" -) - -// Call is a call of a forge tool: the tool by its manifest key and its arguments. -type Call struct { - Tool string - Args map[string]any -} - -// Act is one thing the bridge does on the forge for a key of the gate: the act's name (an owed key's act, `status=`, -// `view`, `note`, `comment.`, or `merge`), the pull request and head it is on (Number 0 for a commit no pull -// request names) and the forge call that performs it. -type Act struct { - Name string - Repository string - Number int - Head string - Call Call -} - -// The forge's tools, by manifest key; their arguments are the ones of mesh-catalog modules/gitea: meshdeliveryclient.ts -// commitStatusTool (owner, repo, sha, context, state, description, target_url), tools/index.ts gitea_delivery_view -// (owner, repo, number, body), gitea_note_append (owner, repo, sha, ref, line), gitea_add_comment (owner, repo, -// number, body), merge.ts gitea_merge_pull_request (owner, repo, number, head, approved_heads). -const ( - commitStatusTool = "gitea.gitea_commit_status" - deliveryViewTool = "gitea.gitea_delivery_view" - noteAppendTool = "gitea.gitea_note_append" - addCommentTool = "gitea.gitea_add_comment" - mergePullTool = "gitea.gitea_merge_pull_request" -) - -// MergeAct is the name of the act of merging a pull request. -const MergeAct = "merge" - -func ownerRepo(repository string) (owner, repo string) { - owner, repo, _ = strings.Cut(repository, "/") - return owner, repo -} - -// OwedAct turns an owed key and its value (mesh-sdk go/delivery OwedKey, EncodeOwed) into the forge call that performs -// it. A key that is no owed key, a value that is no owed act, a value whose act is not the key's, and a view or a comment -// on no pull request are errors, each saying why. -func OwedAct(key string, value []byte) (Act, error) { - repository, number, head, name, err := meshdeliveryclient.ParseOwedKey(key) - if err != nil { - return Act{}, err - } - o, err := meshdeliveryclient.DecodeOwed(value) - if err != nil { - return Act{}, fmt.Errorf("the owed act %s: %w", key, err) - } - kind, context, err := meshdeliveryclient.ParseOwedAct(name) - if err != nil { - return Act{}, err - } - if o.Act != kind { - return Act{}, fmt.Errorf("the owed act %s says %q, not the %q its key names", key, o.Act, kind) - } - owner, repo := ownerRepo(repository) - a := Act{Name: name, Repository: repository, Number: number, Head: head} - needsPull := func() error { - if number <= 0 { - return fmt.Errorf("the owed %s %s is on a commit no pull request names, and it needs the pull request", kind, key) - } - return nil - } - switch kind { - case meshdeliveryclient.OwedStatus: - if o.Context != context { - return Act{}, fmt.Errorf("the owed status %s says the context %q, not the %q its key names", key, o.Context, context) - } - if o.Context == meshdeliveryclient.StatusContextCheck { - // No tool writes mesh/check (R4e finding K1), and the bridge performs no act of it: the forge module writes it from - // the checker's state mesh-delivery-checker.owed, read by a watch (ADR 0521, finding R2-1). - return Act{}, fmt.Errorf("the owed status %s is %s, which the bridge does not perform: the forge module writes it from mesh-delivery-checker.owed", - key, meshdeliveryclient.StatusContextCheck) - } - a.Call = Call{Tool: commitStatusTool, Args: map[string]any{"owner": owner, "repo": repo, "sha": head, "context": o.Context, - "state": o.State, "description": o.Text, "target_url": o.Target}} - case meshdeliveryclient.OwedView: - if err := needsPull(); err != nil { - return Act{}, err - } - a.Call = Call{Tool: deliveryViewTool, Args: map[string]any{"owner": owner, "repo": repo, "number": number, "body": o.Text}} - case meshdeliveryclient.OwedNote: - a.Call = Call{Tool: noteAppendTool, Args: map[string]any{"owner": owner, "repo": repo, "sha": head, "ref": o.Ref, "line": o.Text}} - case meshdeliveryclient.OwedComment: - if err := needsPull(); err != nil { - return Act{}, err - } - a.Call = Call{Tool: addCommentTool, Args: map[string]any{"owner": owner, "repo": repo, "number": number, "body": o.Text}} - } - return a, nil -} - -// MergeActs turns the value of a merges key (the delivery client's EncodeMerge) into the merges to perform: one for each -// pull request in the state merging, in the order the value names them, at the head it expects and with the approved -// heads of every member of the group, `/@` sorted (the merge tool's approved_heads). A pull -// request merged or refused already gets none. A value with no approved heads is an error: the forge's merge tool -// needs them for a group. -func MergeActs(value []byte) ([]Act, error) { - m, err := meshdeliveryclient.DecodeMerge(value) - if err != nil { - return nil, err - } - var handed []seat.MergesByGroupValuePullsItem - for _, p := range m.Pulls { - if meshdeliveryclient.MergeState(p.State) == meshdeliveryclient.MergeMerging { - handed = append(handed, p) - } - } - if len(handed) == 0 { - return nil, nil - } - if len(m.Approved) == 0 { - return nil, fmt.Errorf("the merge of the group %s names no approved heads, so the forge's merge tool has nothing to merge at", m.Group) - } - approved := make([]string, 0, len(m.Approved)) - for key, sha := range m.Approved { - repository, _, ok := strings.Cut(key, "@") - if !ok { - return nil, fmt.Errorf("the approved head %q of the group %s is not @", key, m.Group) - } - approved = append(approved, repository+"@"+sha) - } - slices.Sort(approved) - var acts []Act - for _, p := range handed { - owner, repo := ownerRepo(p.Repository) - acts = append(acts, Act{Name: MergeAct, Repository: p.Repository, Number: int(p.Number), Head: p.Head, - Call: Call{Tool: mergePullTool, Args: map[string]any{"owner": owner, "repo": repo, "number": int(p.Number), "head": p.Head, - "approved_heads": approved}}}) - } - return acts, nil -} - -// RefusedMessage is the refused message the bridge writes on FORGE when the forge refuses an act: the act, the pull -// request and head it was on, and the forge's own words. -func RefusedMessage(a Act, words string) (Message, error) { - event, err := forgemsg.ForgeEvent(a.Repository, forgemsg.ForgeRefused) - if err != nil { - return Message{}, err - } - body, err := forgemsg.EncodeForge(forgemsg.Forge{Kind: forgemsg.ForgeRefused, Repository: a.Repository, Number: a.Number, - Head: a.Head, Act: a.Name, Words: words}) - if err != nil { - return Message{}, err - } - return Message{Event: event, Data: body}, nil -} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go deleted file mode 100644 index a1cb4669..00000000 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/acts_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package bridge - -import ( - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" - "git.novox.be/novox/mesh-sdk/go/keys" -) - -const ( - headA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - headB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" -) - -// owedValue is an owed value as the SDK writes it (mesh-sdk go/delivery EncodeOwed), and its key as OwedKey spells it. -func owedValue(t *testing.T, repository string, number int, act string, o seat.OwedByRepositoryNumberHeadActValue) (string, []byte) { - t.Helper() - key, err := meshdeliveryclient.OwedKey(repository, number, headA, act) - if err != nil { - t.Fatal(err) - } - b, err := meshdeliveryclient.EncodeOwed(o) - if err != nil { - t.Fatal(err) - } - return key, b -} - -func TestOwedStatusIsACommitStatusCall(t *testing.T) { // 1341/AC1, K1 - key, v := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate", - seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: "mesh/merge-gate", State: "pending", Text: "waiting", Target: "https://forge.example/novox/mesh-delivery/pulls/12"}) - a, err := OwedAct(key, v) - if err != nil { - t.Fatal(err) - } - want := Call{Tool: "gitea.gitea_commit_status", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "sha": headA, - "context": "mesh/merge-gate", "state": "pending", "description": "waiting", "target_url": "https://forge.example/novox/mesh-delivery/pulls/12"}} - if !reflect.DeepEqual(a.Call, want) { - t.Errorf("call = %+v, want %+v", a.Call, want) - } - if a.Repository != "novox/mesh-delivery" || a.Number != 12 || a.Head != headA || a.Name != "status=mesh/merge-gate" { - t.Errorf("act = %+v", a) - } - // R4e findings K1 and R2-1 (issue 1301 S4; ADR 0503 §3, ADR 0521): no tool writes mesh/check, since a tool is told no - // caller and node-tools invokes every tool; the forge module writes it from the checker's owed state, and the bridge - // performs no mesh/check act. - ck, cv := checkerKV(t, seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: meshdeliveryclient.StatusContextCheck, State: "success", Text: "pass", Target: "https://view.example/checks/g.1"}) - if a, err := OwedAct(ck, cv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { - t.Fatalf("OwedAct(mesh/check) = %+v, %v; want refused, naming where the forge module reads it", a, err) - } - // Nor through the gate's own owed bucket: the context is refused whichever key carries it. - gk, gv := owedValue(t, "novox/mesh-delivery", 12, "status=mesh/check", - seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedStatus, Context: meshdeliveryclient.StatusContextCheck, State: "success", Text: "pass", Target: "https://x.example"}) - if a, err := OwedAct(gk, gv); err == nil || !strings.Contains(err.Error(), "mesh-delivery-checker.owed") { - t.Fatalf("OwedAct(gate's mesh/check) = %+v, %v; want refused", a, err) - } -} - -// checkerKV is the status the checker owes, as it writes it (the gate's owed key with the act status=mesh/check). -func checkerKV(t *testing.T, o seat.OwedByRepositoryNumberHeadActValue) (string, []byte) { - t.Helper() - key, err := meshdeliveryclient.OwedKey("novox/mesh-delivery", 12, headA, "status=mesh/check") - if err != nil { - t.Fatal(err) - } - v, err := meshdeliveryclient.EncodeOwed(o) - if err != nil { - t.Fatal(err) - } - return key, v -} - -func TestOwedViewNoteAndCommentCalls(t *testing.T) { // 1341/AC2 - cases := []struct { - act string - o seat.OwedByRepositoryNumberHeadActValue - want Call - }{ - {"view", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedView, Text: "the view"}, - Call{Tool: "gitea.gitea_delivery_view", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "body": "the view"}}}, - {"note", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedNote, Text: "a line", Ref: "mesh-plan"}, - Call{Tool: "gitea.gitea_note_append", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "sha": headA, "ref": "mesh-plan", "line": "a line"}}}, - {"comment.2", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "hello"}, - Call{Tool: "gitea.gitea_add_comment", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "body": "hello"}}}, - } - for _, c := range cases { - key, v := owedValue(t, "novox/mesh-delivery", 12, c.act, c.o) - a, err := OwedAct(key, v) - if err != nil { - t.Fatalf("%s: %v", c.act, err) - } - if !reflect.DeepEqual(a.Call, c.want) { - t.Errorf("%s: call = %+v, want %+v", c.act, a.Call, c.want) - } - } -} - -func TestOwedActWithoutAPullRequestNeedsOneOnlyWhereTheToolDoes(t *testing.T) { - key, v := owedValue(t, "novox/mesh-delivery", 0, "note", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedNote, Text: "l", Ref: "mesh-plan"}) - if _, err := OwedAct(key, v); err != nil { - t.Errorf("a note on a commit no pull request names: %v", err) - } - key, v = owedValue(t, "novox/mesh-delivery", 0, "view", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedView, Text: "v"}) - if _, err := OwedAct(key, v); err == nil { - t.Error("a view of no pull request was taken") - } -} - -func TestOwedActThatDisagreesWithItsKeyIsRefused(t *testing.T) { - // A dotted repository's key is unescaped, and its act names the repository whole. - dk, dv := owedValue(t, "novox/my.repo", 3, "comment.1", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "x"}) - if a, err := OwedAct(dk, dv); err != nil || a.Call.Args["repo"] != "my.repo" || a.Repository != "novox/my.repo" { - t.Errorf("a dotted repository's act = %+v, %v", a, err) - } - key, _ := owedValue(t, "novox/mesh-delivery", 12, "view", seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedView, Text: "v"}) - v, _ := meshdeliveryclient.EncodeOwed(seat.OwedByRepositoryNumberHeadActValue{Act: meshdeliveryclient.OwedComment, Text: "v"}) - if _, err := OwedAct(key, v); err == nil { - t.Error("a comment under a view's key was taken") - } - if _, err := OwedAct("nonsense", v); err == nil { - t.Error("a key that is no owed key was taken") - } - if _, err := OwedAct(key, []byte(`{"v":1`)); err == nil { - t.Error("a value that is not json was taken") - } -} - -func mergeValue(t *testing.T, m seat.MergesByGroupValue) []byte { - t.Helper() - b, err := meshdeliveryclient.EncodeMerge(m) - if err != nil { - t.Fatal(err) - } - return b -} - -func TestMergeActsHandOnlyTheMemberHandedWithEveryApprovedHead(t *testing.T) { // 1342/AC1 - approved := map[string]string{meshdeliveryclient.ApprovedKey("novox/mesh-sdk", 4): headB, meshdeliveryclient.ApprovedKey("novox/mesh-delivery", 12): headA} - v := mergeValue(t, seat.MergesByGroupValue{Group: "g", Ask: "merge.g.0123456789ab", Approved: approved, Pulls: []seat.MergesByGroupValuePullsItem{ - {Repository: "novox/mesh-sdk", Number: 4, Head: headB, State: string(meshdeliveryclient.MergeMerged), Commit: "c0"}, - {Repository: "novox/mesh-delivery", Number: 12, Head: headA, State: string(meshdeliveryclient.MergeMerging)}, - }}) - acts, err := MergeActs(v) - if err != nil { - t.Fatal(err) - } - if len(acts) != 1 { - t.Fatalf("%d acts, want 1 (the merged member is not merged again)", len(acts)) - } - want := Call{Tool: "gitea.gitea_merge_pull_request", Args: map[string]any{"owner": "novox", "repo": "mesh-delivery", "number": 12, "head": headA, - "approved_heads": []string{"novox/mesh-delivery@" + headA, "novox/mesh-sdk@" + headB}}} - if !reflect.DeepEqual(acts[0].Call, want) { - t.Errorf("call = %+v, want %+v", acts[0].Call, want) - } - if acts[0].Name != "merge" || acts[0].Number != 12 || acts[0].Head != headA { - t.Errorf("act = %+v", acts[0]) - } - // A merge with no approved heads is refused: the forge's tool would refuse it with nothing to say which. - v = mergeValue(t, seat.MergesByGroupValue{Group: "g", Ask: "a", Pulls: []seat.MergesByGroupValuePullsItem{ - {Repository: "novox/mesh-delivery", Number: 12, Head: headA, State: string(meshdeliveryclient.MergeMerging)}}}) - if _, err := MergeActs(v); err == nil { - t.Error("a merge with no approved heads was taken") - } -} - -func TestRefusedMessageNamesTheActAndTheForgesWords(t *testing.T) { // 1341/AC3, 1342/AC2 - a := Act{Name: "merge", Repository: "novox/mesh-delivery", Number: 12, Head: headA} - m, err := RefusedMessage(a, "novox/mesh-delivery#12 is not merged: its head moved") - if err != nil { - t.Fatal(err) - } - if m.Event != "forge.novox.mesh-delivery.refused" { - t.Errorf("event = %q", m.Event) - } - f, err := forgemsg.DecodeForge(m.Data) - if err != nil { - t.Fatal(err) - } - want := forgemsg.Forge{V: keys.Version, Kind: forgemsg.ForgeRefused, Repository: "novox/mesh-delivery", Number: 12, Head: headA, - Act: "merge", Words: "novox/mesh-delivery#12 is not merged: its head moved"} - if !forgemsg.SameForge(f, want) { - t.Errorf("message = %+v", f) - } -} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go index 344cc9bd..fee308e4 100644 --- a/modules/mesh-delivery-forge-bridge/internal/bridge/call.go +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/call.go @@ -89,7 +89,7 @@ var errNoAnswer = errors.New("the seat git's answer carries nothing") // MergedMessage is the merged message of a merge into the trunk, asked again: built as the bridge builds the one it writes // on FORGE (it is that event's body through ConvertKept), from the pull request's owner, repository, number, branch, head -// and merge commit, which the gate's merges state holds. It reads the forge for every module directory (no kept inputs). +// and merge commit, which the seat git's verb pull answers. It reads the forge for every module directory (no kept inputs). func MergedMessage(owner, repo string, number int, branch, head, merge string, trees Trees, files Files) (Message, error) { body, err := json.Marshal(git.PullMergedEvent{Repository: repository(owner, repo), Number: int64(number), Branch: branch, Head: head, Base: Trunk, MergeCommit: merge}) if err != nil { diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go b/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go new file mode 100644 index 00000000..4fd274da --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/owed.go @@ -0,0 +1,147 @@ +package bridge + +import ( + "encoding/json" + "errors" + "fmt" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + checker "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" +) + +// OwedCall is one act owed to the forge as the seat git's verb that performs it: the seat that said it (Of), the act's +// id, and the verb with its input (novox/hq design 47 *What the gate owes the forge*). Every verb is safe to repeat at +// the forge: a status and the view replace what stands, a merge of a pull request merged at that head is answered +// performed, and a comment and a note line carry the id, which the seat's holder skips when it already holds it. +type OwedCall struct { + Of string + ID string + Act string + Repository string + Number int + Verb string + Input any +} + +// ErrUndecodable is an owed event the bridge cannot read: it is said as owed.ended `undecodable`, never only logged. +var ErrUndecodable = errors.New("an owed act the bridge cannot read") + +func undecodable(why string, args ...any) error { + return fmt.Errorf("%w: %s", ErrUndecodable, fmt.Sprintf(why, args...)) +} + +// GateOwed reads the seat mesh-delivery's event owed into the seat git's verb that performs it. +func GateOwed(data []byte) (OwedCall, error) { + var o seat.OwedEvent + if err := json.Unmarshal(data, &o); err != nil { + return OwedCall{}, undecodable("the gate's owed is not json: %v", err) + } + c := OwedCall{Of: seat.Seat, ID: o.ID, Act: o.Act, Repository: o.Repository, Number: int(o.Number)} + if err := c.common(o.Head); err != nil { + return c, err + } + commit := o.Commit + if commit == "" { + commit = o.Head + } + needsPull := func() error { + if o.Number <= 0 { + return undecodable("the owed %s %s names no pull request", o.Act, o.ID) + } + return nil + } + switch o.Act { + case seat.OwedActMerge: + // Only a switched act is a merge (§169): from G's delivery the gate owes none (§167), a session calls merge-pull + // itself; a merge of the old state, not yet performed by the old bridge, is not dropped. + if err := needsPull(); err != nil { + return c, err + } + c.Verb, c.Input = git.VerbMergePull, git.MergePullInput{ID: o.ID, Repository: o.Repository, Number: o.Number, Head: o.Head} + case seat.OwedActStatus: + if o.Context == "" || o.State == "" { + return c, undecodable("the owed status %s names no context or state", o.ID) + } + c.Verb, c.Input = git.VerbStatus, git.StatusInput{ID: o.ID, Repository: o.Repository, Commit: commit, Context: o.Context, + State: o.State, Description: o.Description, Target: o.Target} + case seat.OwedActView: + if err := needsPull(); err != nil { + return c, err + } + c.Verb, c.Input = git.VerbView, git.ViewInput{ID: o.ID, Repository: o.Repository, Number: o.Number, Text: o.Text} + case seat.OwedActComment: + if err := needsPull(); err != nil { + return c, err + } + c.Verb, c.Input = git.VerbComment, git.CommentInput{ID: o.ID, Repository: o.Repository, Number: o.Number, Text: o.Text} + case seat.OwedActNote: + if o.Line == "" { + return c, undecodable("the owed note line %s says no line", o.ID) + } + c.Verb, c.Input = git.VerbNote, git.NoteInput{ID: o.ID, Repository: o.Repository, Commit: commit, Line: o.Line} + default: + return c, undecodable("the owed %s is the act %q, none the seat git performs", o.ID, o.Act) + } + return c, nil +} + +// CheckerOwed reads the seat mesh-delivery-checker's event owed (its mesh/check status) into the seat git's verb status. +func CheckerOwed(data []byte) (OwedCall, error) { + var o checker.OwedEvent + if err := json.Unmarshal(data, &o); err != nil { + return OwedCall{}, undecodable("the checker's owed is not json: %v", err) + } + c := OwedCall{Of: checker.Seat, ID: o.ID, Act: o.Act, Repository: o.Repository, Number: int(o.Number)} + if err := c.common(o.Head); err != nil { + return c, err + } + if o.Act != checker.OwedActStatus || o.Context == "" || o.State == "" { + return c, undecodable("the checker's owed %s is %q of the context %q in the state %q, not a status", o.ID, o.Act, o.Context, o.State) + } + c.Verb, c.Input = git.VerbStatus, git.StatusInput{ID: o.ID, Repository: o.Repository, Commit: o.Head, Context: o.Context, + State: o.State, Description: o.Description, Target: o.Target} + return c, nil +} + +func (c OwedCall) common(head string) error { + switch { + case c.ID == "": + return undecodable("an owed %s of %s names no id", c.Act, c.Repository) + case c.Repository == "": + return undecodable("the owed %s names no repository", c.ID) + case !sha40.MatchString(head): + return undecodable("the owed %s names the head %q, not 40 hex digits", c.ID, head) + } + if _, _, err := splitRepository(c.Repository); err != nil { + return undecodable("the owed %s: %v", c.ID, err) + } + return nil +} + +// Outcome is what an act's call of the seat git came to, as owed.ended says it: performed when the forge holds the act +// (done now, or its id already held); refused with the forge's own words when the seat answers a refusal or an error +// of the kind refused; could not be performed with the error when the seat did not answer, answered an error of the +// kind failed, or answered what the contract cannot read. +func Outcome(raw json.RawMessage, callErr error) (outcome, words string) { + if callErr != nil { + return forgemsg.OwedCouldNotBePerformed, callErr.Error() + } + var a git.MergePullAnswer // every act verb's answer is this one's fields or fewer + if err := json.Unmarshal(raw, &a); err != nil { + return forgemsg.OwedCouldNotBePerformed, "the seat git's answer is not its contract's: " + err.Error() + } + switch { + case a.Refused != "": + return forgemsg.OwedRefused, a.Refused + case a.Error != "" && a.ErrorKind == git.ErrorKindRefused: + return forgemsg.OwedRefused, a.Error + case a.Error != "": + // failed, or a kind the contract does not name: the forge did not do it, and did not say no + return forgemsg.OwedCouldNotBePerformed, a.Error + case a.Performed || a.Held: + return forgemsg.OwedPerformed, "" + } + return forgemsg.OwedCouldNotBePerformed, "the seat git's answer says neither performed, held, refused nor an error: " + string(raw) +} diff --git a/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go b/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go new file mode 100644 index 00000000..bdc235e6 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/bridge/owed_test.go @@ -0,0 +1,95 @@ +package bridge + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + git "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + + "mesh-delivery-forge-bridge/internal/gitfixture" +) + +// Design 47 *What the gate owes the forge*: each act the gate or the checker says as owed is the seat git's verb of that +// act, under the act's id; what cannot be read is undecodable, and every answer is one outcome. +func TestAnOwedEventIsTheSeatGitsVerbAndItsAnswerOneOutcome(t *testing.T) { + gate := func(name string) []byte { return gitfixture.Seat(t, "mesh-delivery", "events", name, "event:owed")[0] } + chk := gitfixture.Seat(t, "mesh-delivery-checker", "events", "owed is said", "event:owed")[0] + a40 := strings.Repeat("a", 40) + edit := func(raw []byte, k string, v any) []byte { + var m map[string]any + _ = json.Unmarshal(raw, &m) + if v == nil { + delete(m, k) + } else { + m[k] = v + } + b, _ := json.Marshal(m) + return b + } + for _, c := range []struct { + name string + read func([]byte) (OwedCall, error) + data []byte + verb string + input any + bad string + }{ + {"a merge on no pull request", GateOwed, edit(gate("owed is said, a merge"), "number", 0), "", nil, "no pull request"}, + {"a status on the head", GateOwed, gate("owed is said, a status"), "status", git.StatusInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate", + Repository: "novox/mesh-sdk", Commit: a40, Context: "mesh/merge-gate", State: "pending", Description: "waits for the operator's Approve", Target: "https://forge.test/x"}, ""}, + {"a view", GateOwed, edit(edit(gate("owed is said, a status"), "act", "view"), "text", "the view"), "view", + git.ViewInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate", Repository: "novox/mesh-sdk", Number: 178, Text: "the view"}, ""}, + {"a comment", GateOwed, edit(edit(gate("owed is said, a status"), "act", "comment"), "text", "hello"), "comment", + git.CommentInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate", Repository: "novox/mesh-sdk", Number: 178, Text: "hello"}, ""}, + {"a note line on its commit", GateOwed, edit(edit(edit(gate("owed is said, a status"), "act", "note"), "line", "a line"), "commit", strings.Repeat("c", 40)), "note", + git.NoteInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/merge-gate", Repository: "novox/mesh-sdk", Commit: strings.Repeat("c", 40), Line: "a line"}, ""}, + {"the checker's status", CheckerOwed, chk, "status", git.StatusInput{ID: "status.novox/mesh-sdk#178.aaaaaaaaaaaa.mesh/check", + Repository: "novox/mesh-sdk", Commit: a40, Context: "mesh/check", State: "success", Description: "passed", Target: "https://forge.test/x"}, ""}, + {"not json", GateOwed, []byte("{"), "", nil, "not json"}, + {"no id", GateOwed, edit(gate("owed is said, a merge"), "id", nil), "", nil, "no id"}, + {"a short head", CheckerOwed, edit(chk, "head", "abc"), "", nil, "40 hex"}, + {"a merge of the switch", GateOwed, gate("owed is said, a merge"), "merge-pull", git.MergePullInput{ID: "merge.feat=47-G.0123456789ab.novox/mesh-sdk#178", + Repository: "novox/mesh-sdk", Number: 178, Head: a40}, ""}, + {"a comment on no pull request", GateOwed, edit(edit(gate("owed is said, a status"), "act", "comment"), "number", 0), "", nil, "no pull request"}, + {"an act of no kind", GateOwed, edit(gate("owed is said, a status"), "act", "label"), "", nil, "none the seat git"}, + {"a repository not owner/name", GateOwed, edit(gate("owed is said, a status"), "repository", "mesh-sdk"), "", nil, "owner"}, + } { + got, err := c.read(c.data) + switch { + case c.bad != "": + if !errors.Is(err, ErrUndecodable) || !strings.Contains(err.Error(), c.bad) { + t.Errorf("%s: %v, want undecodable naming %q", c.name, err, c.bad) + } + case err != nil || got.Verb != c.verb || !reflect.DeepEqual(got.Input, c.input): + t.Errorf("%s: %s %+v %v, want %s %+v", c.name, got.Verb, got.Input, err, c.verb, c.input) + } + } + answers := func(verb string) []json.RawMessage { return gitfixture.All(t, verb+" answers", "verb:"+verb+".answer") } + failed := func(verb string) json.RawMessage { + return gitfixture.Given(t, verb+" answers an error, never none", "verb:"+verb+".answer") + } + for _, c := range []struct { + name string + raw json.RawMessage + err error + outcome, words string + }{ + {"a merge performed", answers("merge-pull")[0], nil, forgemsg.OwedPerformed, ""}, + {"a merge refused", answers("merge-pull")[1], nil, forgemsg.OwedRefused, "the head moved"}, + {"a comment already held", answers("comment")[0], nil, forgemsg.OwedPerformed, ""}, + {"the forge's error, of the kind refused", failed("comment"), nil, forgemsg.OwedRefused, "403"}, + {"the forge's error, of the kind failed", failed("view"), nil, forgemsg.OwedCouldNotBePerformed, "502"}, + {"no answer in the call's bound", nil, errors.New("nats: timeout"), forgemsg.OwedCouldNotBePerformed, "timeout"}, + {"an answer saying nothing", json.RawMessage(`{"id":"x"}`), nil, forgemsg.OwedCouldNotBePerformed, "neither"}, + {"an answer not the contract's", json.RawMessage(`[1]`), nil, forgemsg.OwedCouldNotBePerformed, "contract"}, + } { + outcome, words := Outcome(c.raw, c.err) + if outcome != c.outcome || !strings.Contains(words, c.words) { + t.Errorf("%s: %q %q, want %q naming %q", c.name, outcome, words, c.outcome, c.words) + } + } +} diff --git a/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go b/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go index f3e5f09d..29641622 100644 --- a/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go +++ b/modules/mesh-delivery-forge-bridge/internal/gitfixture/gitfixture.go @@ -13,9 +13,16 @@ import ( // All is every fixture's body named name of of (such as "file answers", "verb:file.answer"), in the file's order; a // name with none fails the test. func All(t testing.TB, name, of string) []json.RawMessage { + t.Helper() + return Seat(t, "git", "verbs", name, of) +} + +// Seat is every fixture's body named name of of in a seat's fixtures file (such as "mesh-delivery", "events"), so the +// bridge's fakes of the seats it hears say what those seats say. +func Seat(t testing.TB, seat, file, name, of string) []json.RawMessage { t.Helper() _, here, _, _ := runtime.Caller(0) - raw, err := os.ReadFile(filepath.Join(filepath.Dir(here), "../../../../contracts/seats/git/fixtures/verbs.json")) + raw, err := os.ReadFile(filepath.Join(filepath.Dir(here), "../../../../contracts/seats", seat, "fixtures", file+".json")) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/groups.go b/modules/mesh-delivery-forge-bridge/internal/groups/groups.go index e3ddbc03..5cfde96d 100644 --- a/modules/mesh-delivery-forge-bridge/internal/groups/groups.go +++ b/modules/mesh-delivery-forge-bridge/internal/groups/groups.go @@ -5,6 +5,7 @@ package groups import ( + "context" "crypto/sha256" "encoding/hex" "encoding/json" @@ -12,12 +13,13 @@ import ( "fmt" "slices" "sort" - "strconv" "strings" "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/sdk/trunk" + "git.novox.be/novox/mesh-sdk/go/keys" ) // The bridge's states the tools read and write. @@ -26,20 +28,34 @@ const ( Module = "mesh-delivery-forge-bridge" // PerformedState is the bridge's state of what it performed and published. PerformedState = "performed" - // EndAsksState is the state of asks to say a group's end again, keyed by the gate's merges key of the group. + // EndAsksState is the state of asks to say a group's end again, keyed by the group escaped as a key. EndAsksState = "end-asks" // EndAsksBucket is its bucket on the bus. EndAsksBucket = Module + "_" + EndAsksState ) -// Key is the key of performed that says a group's end is written: `groups.`, its value the revision of the -// gate's merges key it was worked out from. -func Key(mergesKey string) string { return "groups." + mergesKey } +// Key is the key of performed that says a group's end is written: `groups.`, its value the last +// member's merge commit the end named. +func Key(group string) (string, error) { + part, err := KeyOf(group) + if err != nil { + return "", err + } + return "groups." + part, nil +} + +// KeyOf is a group's id (its branch) escaped as a key, as the states performed and end-asks key it. +func KeyOf(group string) (string, error) { + if strings.TrimSpace(group) == "" { + return "", errors.New("a group's id is empty") + } + return keys.EscapeName(group), nil +} // PublishedKey is the key of performed that says a member's merged message is written: `published.<16 hex digits of the -// sha256 of #@>`. -func PublishedKey(repository string, number int, merge string) string { - sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d@%s", strings.ToLower(repository), number, merge))) +// sha256 of #>`, its value the merge commit the message named (a pull request merges once). +func PublishedKey(repository string, number int) string { + sum := sha256.Sum256([]byte(fmt.Sprintf("%s#%d", strings.ToLower(repository), number))) return "published." + hex.EncodeToString(sum[:8]) } @@ -72,13 +88,13 @@ type EndAsk struct { // Done says the ask is answered. func (a EndAsk) Done() bool { return !a.Answered.IsZero() } -// NewEndAsk is the value bridge_group_end_again writes for a group: its key (the gate's merges key) and the value. +// NewEndAsk is the value bridge_group_end_again writes for a group: its key (the group escaped as a key) and the value. func NewEndAsk(group, why, by string, at time.Time) (string, []byte, error) { group = strings.TrimSpace(group) if group == "" { return "", nil, errors.New("name the group whose end is said again (group)") } - key, err := meshdeliveryclient.MergesKey(group) + key, err := KeyOf(group) if err != nil { return "", nil, err } @@ -98,21 +114,24 @@ func DecodeEndAsk(data []byte) (EndAsk, error) { return a, nil } -// Row is one group as bridge_groups answers it: the gate's merge of it and what the bridge kept of its end. +// Row is one group as bridge_groups answers it: the seat mesh-delivery's group and what the bridge kept of its end. type Row struct { Key string `json:"key"` Group string `json:"group"` + State string `json:"state,omitempty"` Members []RowMember `json:"members"` Ended bool `json:"ended"` - EndedAt uint64 `json:"ended-at-revision,omitempty"` - Open bool `json:"open"` - Words string `json:"words"` + // EndedWith is the last member's merge commit the end named, when it is written. + EndedWith string `json:"ended-with,omitempty"` + Open bool `json:"open"` + Words string `json:"words"` // LastEndAsk is the last ask to say the group's end again (bridge_group_end_again), its who, why and when, and the // bridge's answer once it read it; absent when none was asked. The state end-asks keeps the last per group. LastEndAsk *EndAsk `json:"last-end-ask,omitempty"` } -// RowMember is one approved member of a group: its merge's state at the gate and whether its merged message is written. +// RowMember is one member of a group: its delivery state at the seat and, once its merged message is written, the merge +// commit it named. type RowMember struct { Repository string `json:"repository"` Number int `json:"number"` @@ -121,12 +140,12 @@ type RowMember struct { Published bool `json:"published"` } -// Rows are the groups the gate's merges hold with more than one member, each with what the bridge kept: ended (its end -// written, the revision it was worked out from) or open (some member is not merged or its merged message not written, or its -// end not written). merges is the gate's merges by key, performed the bridge's performed keys by key; unread are the gate's -// keys that do not read. asks are the end-asks values by the same key; one that does not read is named unread as -// `end-asks.`. -func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[string]uint64, asks map[string][]byte) (rows []Row, unread []string) { +// Rows are the groups the seat mesh-delivery's verb groups answers, each with what the bridge kept: ended (its end written, +// with the last merge it named) or open (some member's merged message is not written, or its end not written). published +// is performed's `published.` values by key (the merge commit), ended its `groups.` values by key; asks are the end-asks +// values by the same key, and one that does not read is named unread as `end-asks.`. A group whose id cannot be a +// key is named unread by its id. +func Rows(of []seat.Group, published, ended map[string]string, asks map[string][]byte) (rows []Row, unread []string) { lastAsk := map[string]*EndAsk{} for key, raw := range asks { a, err := DecodeEndAsk(raw) @@ -136,37 +155,32 @@ func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[strin } lastAsk[key] = &a } - for key, raw := range merges { - m, err := meshdeliveryclient.DecodeMerge(raw) + for _, g := range of { + if len(g.Members) < 2 { + continue + } + key, err := KeyOf(g.ID) if err != nil { - unread = append(unread, key) + unread = append(unread, "a group with no id") continue } - if len(m.Approved) < 2 { - continue + r := Row{Key: key, Group: g.ID, State: g.State} + allPublished := true + for _, m := range g.Members { + commit, pub := published[PublishedKey(m.Repository, int(m.Number))] + r.Members = append(r.Members, RowMember{Repository: m.Repository, Number: int(m.Number), State: m.State, Commit: commit, Published: pub}) + allPublished = allPublished && pub } - r := Row{Key: key, Group: m.Group} - allMerged := true - for _, p := range m.Pulls { - pub := p.Commit != "" && performed[PublishedKey(p.Repository, int(p.Number), p.Commit)] - r.Members = append(r.Members, RowMember{Repository: p.Repository, Number: int(p.Number), State: p.State, Commit: p.Commit, Published: pub}) - if meshdeliveryclient.MergeState(p.State) != meshdeliveryclient.MergeMerged || !pub { - allMerged = false - } - } - if len(m.Pulls) < len(m.Approved) { - allMerged = false - } - r.EndedAt, r.Ended = endedAt[key] + r.EndedWith, r.Ended = ended["groups."+key] r.LastEndAsk = lastAsk[key] r.Open = !r.Ended switch { case r.Ended: r.Words = "its end is written" - case allMerged: - r.Words = "every member is merged and its message written, and its end is not written: bridge_group_end_again says it" + case allPublished: + r.Words = "every member's merged message is written, and its end is not written: bridge_group_end_again says it" default: - r.Words = "gathered: not every member is merged and its merged message written" + r.Words = "gathered: not every member's merged message is written" } rows = append(rows, r) } @@ -177,25 +191,46 @@ func Rows(merges map[string][]byte, performed map[string]bool, endedAt map[strin func sortRows(rows []Row) { sort.Slice(rows, func(i, j int) bool { return rows[i].Key < rows[j].Key }) } -// MembersOf are a merge's approved members with their heads, in the order of their keys. -func MembersOf(m seat.MergesByGroupValue) []meshdeliveryclient.Head { - var out []meshdeliveryclient.Head - for _, k := range slices.Sorted(func(yield func(string) bool) { - for k := range m.Approved { - if !yield(k) { - return - } - } - }) { - at := strings.LastIndex(k, "@") - if at < 0 { - continue - } - n, err := strconv.Atoi(k[at+1:]) - if err != nil { - continue - } - out = append(out, meshdeliveryclient.Head{Repository: k[:at], Number: n, Head: m.Approved[k]}) +// MembersOf are a group's members with their heads, in the group's order. +func MembersOf(g seat.Group) []meshdeliveryclient.Head { + out := make([]meshdeliveryclient.Head, 0, len(g.Members)) + for _, m := range g.Members { + out = append(out, meshdeliveryclient.Head{Repository: m.Repository, Number: int(m.Number), Head: m.Head}) } return out } + +// Named is the group of id among the groups the seat answered, when it has more than one member and names the pull +// request number of repository among them (a lone pull request is no group). +func Named(of []seat.Group, id, repository string, number int) (seat.Group, bool) { + for _, g := range of { + if g.ID != id || len(g.Members) < 2 { + continue + } + if slices.ContainsFunc(g.Members, func(m seat.Member) bool { + return trunk.SameRepository(m.Repository, repository) && int(m.Number) == number + }) { + return g, true + } + } + return seat.Group{}, false +} + +// Read is every delivery group the seat mesh-delivery's verb groups answers, the ended ones too (all), called through call +// by the verb's manifest key: the bridge reads a group from the seat, never from the gate's state (design 47 *A group's +// merges are taken in together*). An answer saying an error, or one the contract cannot type, is an error: a failed read +// is never read as no group. +func Read(ctx context.Context, call func(ctx context.Context, key string, body any) (json.RawMessage, error)) ([]seat.Group, error) { + raw, err := call(ctx, "seat:"+seat.Seat+"."+seat.VerbGroups, seat.GroupsInput{All: seat.GroupsInputAllTrue}) + if err != nil { + return nil, fmt.Errorf("the seat %s did not answer %s: %w", seat.Seat, seat.VerbGroups, err) + } + var a seat.GroupsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the seat %s's answer of %s is not its contract's: %w", seat.Seat, seat.VerbGroups, err) + } + if a.Error != "" { + return nil, fmt.Errorf("the seat %s refused %s: %s", seat.Seat, seat.VerbGroups, a.Error) + } + return a.Groups, nil +} diff --git a/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go b/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go new file mode 100644 index 00000000..e8ec3319 --- /dev/null +++ b/modules/mesh-delivery-forge-bridge/internal/groups/owedask.go @@ -0,0 +1,95 @@ +package groups + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strings" + "time" +) + +// What a person asks of the bridge's performing of what is owed (novox/hq design 47 *What the gate owes the forge*): the +// tools bridge_owed (its consumers' backlog), bridge_owed_again and bridge_owed_drop write an ask into the state +// owed-asks and wait for the bridge's answer there, as bridge_group_end_again does, since the bridge is the other program +// and serves no request subject (ADR 0437 §4). +const ( + // OwedAsksState is the state of asks about what is owed, one key per kind of ask and act. + OwedAsksState = "owed-asks" + // OwedAsksBucket is its bucket on the bus. + OwedAsksBucket = Module + "_" + OwedAsksState +) + +// The kinds of an owed ask. +const ( + OwedAgain = "again" + OwedDrop = "drop" + OwedBacklog = "backlog" +) + +// Backlog is one durable consumer of a seat's stream owed as the bridge reads it: the last event applied (its stream +// sequence), the events waiting and those delivered and not yet acknowledged, and the oldest of them. +type Backlog struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + LastApplied uint64 `json:"last-applied"` + Waiting uint64 `json:"waiting"` + Held uint64 `json:"held"` + Oldest time.Time `json:"oldest,omitzero"` + OldestSecs int64 `json:"oldest-secs,omitempty"` + Error string `json:"error,omitempty"` +} + +// OwedAsk is a value of owed-asks: what was asked (of the act Of said under ID, for again and drop), why, who and when, +// and once the bridge answered, when, what it came to (the outcome said as owed.ended) and its words; a backlog ask is +// answered with the consumers' backlog. +type OwedAsk struct { + Kind string `json:"kind"` + Of string `json:"of,omitempty"` + ID string `json:"id,omitempty"` + Why string `json:"why,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Answered time.Time `json:"answered,omitzero"` + Outcome string `json:"outcome,omitempty"` + Answer string `json:"answer,omitempty"` + Backlog []Backlog `json:"backlog,omitempty"` +} + +// Done says the bridge answered it. +func (a OwedAsk) Done() bool { return !a.Answered.IsZero() } + +// NewOwedAsk is an ask's key and value: a backlog ask under the key backlog, an act's under its kind, its seat and its id +// (unpadded base64url, as an id may hold what a key may not). A drop says why. +func NewOwedAsk(kind, of, id, why, by string, at time.Time) (string, []byte, error) { + a := OwedAsk{Kind: kind, Of: of, ID: id, Why: strings.TrimSpace(why), By: by, At: at.UTC()} + key := kind + switch kind { + case OwedBacklog: + a.Of, a.ID = "", "" + case OwedAgain, OwedDrop: + if of == "" || id == "" { + return "", nil, fmt.Errorf("an owed act is named by the seat that said it and its id") + } + if kind == OwedDrop && a.Why == "" { + return "", nil, fmt.Errorf("a drop says why") + } + key = kind + "." + of + "." + base64.RawURLEncoding.EncodeToString([]byte(id)) + default: + return "", nil, fmt.Errorf("no owed ask %q: again, drop or backlog", kind) + } + v, err := json.Marshal(a) + return key, v, err +} + +// DecodeOwedAsk reads a value of owed-asks. +func DecodeOwedAsk(data []byte) (OwedAsk, error) { + var a OwedAsk + if err := json.Unmarshal(data, &a); err != nil { + return a, err + } + switch a.Kind { + case OwedAgain, OwedDrop, OwedBacklog: + return a, nil + } + return a, fmt.Errorf("no owed ask %q", a.Kind) +} diff --git a/modules/mesh-delivery-forge-bridge/manifest_test.go b/modules/mesh-delivery-forge-bridge/manifest_test.go index 7a786eaa..034778b3 100644 --- a/modules/mesh-delivery-forge-bridge/manifest_test.go +++ b/modules/mesh-delivery-forge-bridge/manifest_test.go @@ -27,6 +27,14 @@ type state struct { History int `json:"history"` } +// kept is an entry of requires in its long form: kept data on the bus (novox/hq ADR 0595). +type kept struct { + Name string `json:"name"` + Storage string `json:"storage"` + Template string `json:"template"` + Options map[string]int `json:"options"` +} + type artifact struct { Name string `json:"name"` Kind string `json:"kind"` @@ -46,6 +54,7 @@ type manifest struct { Consumes []string `json:"consumes"` Streams []stream `json:"streams"` State []state `json:"state"` + Requires []kept `json:"requires"` Reads []string `json:"reads"` Tools []string `json:"tools"` Invokes []string `json:"invokes"` @@ -87,20 +96,22 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if m.Module != "mesh-delivery-forge-bridge" { t.Errorf("module = %q", m.Module) } - if !reflect.DeepEqual(m.Streams, []stream{{Name: "forge", Events: []string{"forge.**"}}}) { + if !reflect.DeepEqual(m.Streams, []stream{{Name: "forge", Events: []string{"forge.**", "owed.ended"}}}) { t.Errorf("streams = %+v", m.Streams) } - // The checker's owed is the forge module's to write (ADR 0521, R4e finding R2-1): the bridge neither hears nor reads it. - // The forge's events are the seat git's (design 47 *The seat git*); forge.delete: a branch's end, said as a + // What the gate and the checker owe the forge is each seat's event owed, heard on its own stream (design 47 *What the + // gate owes the forge*), never read from their state. The forge's events are the seat git's (design 47 *The seat git*); forge.delete: a branch's end, said as a // branch-deleted kind (design 51, DLV-FB1). - if want := []string{"git.forge-delete", "git.pull-closed", "git.pull-merged", "git.pull-updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + if want := []string{"git.forge-delete", "git.pull-closed", "git.pull-merged", "git.pull-updated", "mesh-delivery-checker.owed", "mesh-delivery.owed"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } - if want := []string{"mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) { - t.Errorf("reads = %v, want %v", m.Reads, want) + // A group is read from the seat mesh-delivery's verb groups (design 47, acceptance 49): the bridge reads no state of + // another module's. + if len(m.Reads) != 0 { + t.Errorf("reads = %v, want none", m.Reads) } - if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_delivery_view", "gitea.gitea_merge_pull_request", - "gitea.gitea_note_append", "seat:git.file", "seat:git.open-pulls", "seat:git.pull", "seat:git.tags", "seat:git.trees"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + if want := []string{"seat:git.comment", "seat:git.file", "seat:git.merge-pull", "seat:git.note", "seat:git.open-pulls", + "seat:git.pull", "seat:git.status", "seat:git.tags", "seat:git.trees", "seat:git.view", "seat:mesh-delivery.groups", "seat:mesh-hand-acts.hand-act"}; !reflect.DeepEqual(sorted(m.Invokes), want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -109,19 +120,22 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1 if m.SecretsOwner != "mesh-delivery-forge-bridge" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // Three buckets: the head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443 - // §5), the revision of each key of the gate's buckets the bridge performed, so a key is performed once per - // revision (ADR 0437 §8; issue 1341), and what a merge read of each module directory by its tree, so a merge reads - // the forge only for the directories it changed (issue 1126). - if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "performed", History: 1}, {Name: "inputs", History: 1}, {Name: "end-asks", History: 1}}) { // 1279/AC6, 1341, 1126, 1585 - t.Errorf("state = %+v, want the buckets heads, performed, inputs and end-asks", m.State) + // The head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443 §5), and what a + // merge read of each module directory by its tree, so a merge reads the forge only for the directories it changed + // (issue 1126). What it performed of what is owed is kept data in the new form, one key per act, kept 7 days as the + // owners' streams owed are; its size is the template's own (design 47 *What the gate owes the forge*, ADR 0595 §3). + if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}, {Name: "inputs", History: 1}, {Name: "end-asks", History: 1}, {Name: "owed-asks", History: 1}}) { // 1279/AC6, 1126, 1585 + t.Errorf("state = %+v, want the buckets heads, inputs, end-asks and owed-asks", m.State) + } + if want := []kept{{Name: "performed", Storage: "key-value", Template: "state", Options: map[string]int{"history": 1, "ttl-seconds": 604800}}}; !reflect.DeepEqual(m.Requires, want) { + t.Errorf("requires = %+v, want performed as %+v", m.Requires, want) } // 1585/T18: the tools read the groups and ask the bridge to say a group's end again, through its state end-asks. The tool bridge_merged_message answers a merge's merged message again (issue 1585); bridge_tags reads a package's tags for the builder's floor (design 51). - if want := []string{"bridge_group_end_again", "bridge_groups", "bridge_merged_message", "bridge_tags"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"bridge_group_end_again", "bridge_groups", "bridge_merged_message", "bridge_owed", "bridge_owed_again", "bridge_owed_drop", "bridge_tags"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } - // No other state, no bus, no seat, no event (ADR 0437 §4; ADR 0521: no check.status). - for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats"} { + // No other state, no seat, no event (ADR 0437 §4; ADR 0521: no check.status); its kept data on the bus is the one entry of requires above. + for _, k := range []string{"logs", "emits", "claims", "provides", "uses", "defines-seats", "seat", "seats"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the bridge has none", k) } diff --git a/modules/mesh-delivery-forge-bridge/module.json b/modules/mesh-delivery-forge-bridge/module.json index 0105502f..c26d1d23 100644 --- a/modules/mesh-delivery-forge-bridge/module.json +++ b/modules/mesh-delivery-forge-bridge/module.json @@ -6,39 +6,50 @@ "git.pull-updated", "git.pull-merged", "git.pull-closed", - "git.forge-delete" + "git.forge-delete", + "mesh-delivery.owed", + "mesh-delivery-checker.owed" ], "streams": [ - { "name": "forge", "events": ["forge.**"] } + { "name": "forge", "events": ["forge.**", "owed.ended"] } ], "state": [ { "name": "heads", "history": 1 }, - { "name": "performed", "history": 1 }, { "name": "inputs", "history": 1 }, - { "name": "end-asks", "history": 1 } + { "name": "end-asks", "history": 1 }, + { "name": "owed-asks", "history": 1 } ], "tools": [ "bridge_tags", "bridge_groups", "bridge_group_end_again", - "bridge_merged_message" - ], - "reads": [ - "mesh-delivery-gate.merges", - "mesh-delivery-gate.owed" + "bridge_merged_message", + "bridge_owed", + "bridge_owed_again", + "bridge_owed_drop" ], "invokes": [ - "gitea.gitea_merge_pull_request", - "gitea.gitea_commit_status", - "gitea.gitea_delivery_view", - "gitea.gitea_note_append", - "gitea.gitea_add_comment", + "seat:git.status", + "seat:git.comment", + "seat:git.view", + "seat:git.note", + "seat:git.merge-pull", + "seat:mesh-delivery.groups", + "seat:mesh-hand-acts.hand-act", "seat:git.open-pulls", "seat:git.trees", "seat:git.file", "seat:git.pull", "seat:git.tags" ], + "requires": [ + { + "name": "performed", + "storage": "key-value", + "template": "state", + "options": { "history": 1, "ttl-seconds": 604800 } + } + ], "own-secrets": { "broker": "${dir:state}/broker" }, From 91a9c4129056bcd4bdd038074f72eccd5b657ecd Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:55:02 +0100 Subject: [PATCH 15/16] gofmt --- modules/mesh-delivery-assigner/manifest_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 036dbc34..1db630c4 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -127,7 +127,7 @@ func TestManifest(t *testing.T) { t.Errorf("consumes = %v, want the indexer's views of graph, manifests and modules, the teller's ask-ended and the engines' reads changed", m.Consumes) } if want := []string{"mesh-delivery-indexer.graph", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", - "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list + "seat:mesh-catalog.registrations", "seat:mesh-catalog.seat-definitions", // ADR 0636 §4: the seat list "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone "seat:node-backup.backed-up", "seat:node-backup.now", "seat:node-engine.manifest-reads", // ADR 0618 §3 From b1fcabcebe549e550546bd803fadef16889fe22c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:57:53 +0100 Subject: [PATCH 16/16] AS3: a placement's branch and build, set by a proposal, bound by the ask, and its build a held put MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A proposal sets a module's watched branch and build (a re-place keeps them), the changes digest an ask binds to carries both, and a placement's build is a held put of that build, released when the placement drops it while a person's put stays. Taken over from 6bc6fe53 and b8e50fff. Builds: design 47 G §14, §78, §114, §360, §361 Tests: TestAssignments (place change branch, build, digest rows), aPlacementsBuildIsPut. Test 77 (§79) covered by the put row. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-assigner/pass_test.go | 58 +++++++++++ .../mesh-delivery-assigner/placementbuild.go | 95 +++++++++++++++++++ .../cmd/mesh-delivery-assigner/puts.go | 1 + .../internal/assignment/assignments_test.go | 18 ++++ .../internal/assignment/digest.go | 8 +- .../internal/assignment/proposal.go | 10 +- 6 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placementbuild.go diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index 0bc60e9a..5e8e1ce4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -272,6 +272,7 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { t.Error("an answer to an ask taken away was taken") } t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus) + t.Run("a placement's build is put held, and released when the placement names it no more", aPlacementsBuildIsPut) // ADR 0614 §4: the rules read what each node's engine states it reads from its record in `.node`; a node with no // record, or one that states nothing, is read as stating nothing (the rules then read the baseline). t.Run("the rules read what each node's engine reads", func(t *testing.T) { @@ -467,6 +468,63 @@ func (b *fakeBus) Emit(eventType string, body any) error { return nil } +// Design 47 line 912 and line 914 ("a placement's `build` delivers the earlier build"; ADR 0666 §24, B-K1): a placed +// module's build becomes a held put of that build, which then goes a put's steps; a placement naming no build any more +// releases the put it made, never a person's. +func aPlacementsBuildIsPut(t *testing.T) { + ctx := context.Background() + putsKV := &memKV{values: map[string][]byte{}} + p := newPasser(kvSet{buckets.Puts: putsKV}, time.Date(2026, 11, 10, 18, 0, 0, 0, time.UTC)) + earlier := meshcatalogclient.SourcePrefix2 + strings.Repeat("e", 64) + place := func(build string) { + values, err := mmac.AssignmentsValues(mma.Assignments{Node: "g14", Modules: []mma.PlacedModule{{Module: "web", Build: build}, {Module: "db"}}}) + if err != nil { + t.Fatal(err) + } + for key, raw := range values { + p.w.putEntry(plc.AssignmentsBucket, key, raw, uint64(len(raw)), time.Now()) + } + } + read := func() (puts.Put, bool) { + raw, ok := putsKV.values["g14.web"] + if !ok { + return puts.Put{}, false + } + got, err := puts.Decode(raw) + if err != nil { + t.Fatal(err) + } + return got, true + } + place(earlier) + p.placementPuts(ctx) + got, ok := read() + if !ok || got.FP != earlier || !got.Hold || got.By != placementBy || got.Step != puts.Asked { + t.Fatalf("the placement's build made the put %+v (there %v)", got, ok) + } + if _, there := putsKV.values["g14.db"]; there { + t.Error("a placed module naming no build was put") + } + p.placementPuts(ctx) // again: the same build makes nothing new + if again, _ := read(); !again.At.Equal(got.At) || again.Step != puts.Asked { + t.Errorf("a second pass made the put again: %+v", again) + } + place("") + p.placementPuts(ctx) + if got, _ := read(); got.Step != puts.Released || !strings.Contains(got.Words, placementBy) { + t.Errorf("the placement naming no build left the put %+v", got) + } + // A person's put stands while the placement names no build. + person := puts.Put{FP: earlier, Hold: true, Why: "a try", By: "lead@shanks", At: time.Now()}.Said(time.Now(), puts.Holding, "held") + raw, _ := puts.Encode(person) + putsKV.values["g14.web"] = raw + p.w.put(buckets.Puts, "g14.web", raw) + p.placementPuts(ctx) + if got, _ := read(); got.Step != puts.Holding || got.By != "lead@shanks" { + t.Errorf("a person's put was released by the placement: %+v", got) + } +} + // ADR 0546, the first put (2026-10-25, want.novox.nats): the tool writes the put; a pass waits for the composition and grant // the controller's take-in writes; once they are there it asks the operator with a yes that approves; only on that warrant does it have // the streams snapshotted, then writes the want held and records the warrant; the have running it healthy holds it; a release diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placementbuild.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placementbuild.go new file mode 100644 index 00000000..3fb5d39c --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/placementbuild.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "fmt" + "sort" + + "mesh-delivery-assigner/internal/buckets" + "mesh-delivery-assigner/internal/puts" +) + +// placementBy is who a put the placement's build makes names as its asker: the placement, whose proposal the operator +// approved, is the warrant (design 47 *Going back is one act*; ADR 0666 §24). +const placementBy = "the placement" + +// placementPuts makes each placed module's `build` a held put of that build, and releases a put the placement made once its +// placement names that build no more (design 47 line 912: "a return to an earlier build is one value in the placement, the +// module's `build` in its placement key, set by a placement proposal"; ADR 0666 §24; B-K1). The put then goes its own steps +// (the builder's build, the composition, an ask where the module's policy needs one, the want, holding) and is read with +// `assigner_puts` and released with `assigner_release` like any other; a placement's build the builder never built is refused +// by its build step, said in the put's words. A put asked by a person stands until the placement names another build. +func (p *passer) placementPuts(ctx context.Context) { + kv := p.kvs[buckets.Puts] + if kv == nil { + return + } + now := p.clock().UTC() + named := map[string]string{} // put key -> the build its placement names + for node, a := range p.w.assignments() { + for _, m := range a.Modules { + if m.Build == "" { + continue + } + if key, err := puts.Key(node, m.Module); err == nil { + named[key] = m.Build + } + } + } + held := p.w.read(buckets.Puts) + keys := make([]string, 0, len(named)+len(held)) + for k := range named { + keys = append(keys, k) + } + for k := range held { + if _, there := named[k]; !there { + keys = append(keys, k) + } + } + sort.Strings(keys) + for _, key := range keys { + build, names := named[key] + var before *puts.Put + if raw, ok := held[key]; ok { + if got, err := puts.Decode(raw); err == nil { + before = &got + } + } + var next puts.Put + switch { + case names && before != nil && before.FP == build: + continue // made already, or a person's put of the same build: it goes its own steps, ended or not + case names: + words := fmt.Sprintf("asked by %s: it names build %s of %s", placementBy, shortFP(build), key) + if before != nil && !puts.Ended(before.Step) { + words += fmt.Sprintf("; it replaces the put of %s, which stood at %s", shortFP(before.FP), before.Step) + } + next = puts.Put{FP: build, Hold: true, Why: "the placement names this build (ADR 0666 §24)", By: placementBy, At: now}.Said(now, puts.Asked, words) + case before != nil && before.By == placementBy && !puts.Ended(before.Step): + next = before.Said(now, puts.Released, fmt.Sprintf("released by %s: it names no build of its own any more; the assigner's rules write the want again", placementBy)) + default: + continue + } + data, err := puts.Encode(next) + if err != nil { + p.logf("the placement's put %s is not written: %v", key, err) + continue + } + if before == nil { + _, err = kv.Create(ctx, key, data) + } else if e, gerr := kv.Get(ctx, key); gerr != nil { + err = gerr + } else { + _, err = kv.Update(ctx, key, data, e.Revision()) + } + if err != nil { + p.logf("the placement's put %s could not be written, and is written at the next pass: %v", key, err) + continue + } + p.w.put(buckets.Puts, key, data) + p.logf("put %s: %s: %s", key, next.Step, next.Words) + if p.acts != nil { + p.acts.say(key, next) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go index 929ecbdb..77ccfaf2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/puts.go @@ -105,6 +105,7 @@ func (p *passer) puts(ctx context.Context) { if p.acts == nil { return } + p.placementPuts(ctx) all := p.putsByModule() if len(all) == 0 { return diff --git a/modules/mesh-delivery-assigner/internal/assignment/assignments_test.go b/modules/mesh-delivery-assigner/internal/assignment/assignments_test.go index 13789020..33a30427 100644 --- a/modules/mesh-delivery-assigner/internal/assignment/assignments_test.go +++ b/modules/mesh-delivery-assigner/internal/assignment/assignments_test.go @@ -129,6 +129,24 @@ func TestAssignments(t *testing.T) { if !reflect.DeepEqual(client.Assigned(current["g14"]), []string{"zsh", "app"}) { t.Error("laying changed the current assignments") } + // Design 47 line 1051 (the Choices table: a module's watched branch and its build in the placement key, set by a + // proposal): a place change sets them, and a re-place naming none keeps the module's. + onBeta, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Branch: "beta", Build: "b1"}}) + again, g := LayChanges(onBeta, []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Ports: map[string]int64{"80": 8081}}}) + if m := onBeta["g14"].Modules[0]; len(f) != 0 || m.Branch != "beta" || m.Build != "b1" { + t.Errorf("a place change's branch and build: %+v, %v", m, f) + } + if m := again["g14"].Modules[0]; len(g) != 0 || m.Branch != "beta" || m.Build != "b1" || m.Ports["80"] != 8081 { + t.Errorf("a re-place dropped the branch and build: %+v, %v", m, g) + } + if ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Branch: "beta"}}}) == + ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh"}}}) { + t.Error("an ask bound to a place change's digest does not bind its branch") + } + if ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh", Build: "b1"}}}) == + ChangesDigest(seat.Proposal{Changes: []seat.ProposalChange{{Op: client.OpPlace, Node: "g14", Module: "zsh"}}}) { + t.Error("an ask bound to a place change's digest does not bind its build (design 47 G 360)") + } if fresh, f := LayChanges(current, []seat.ProposalChange{{Op: client.OpPlace, Node: "pi", Module: "zsh"}}); len(f) != 0 || fresh["pi"].Node != "pi" { t.Errorf("a node with no key: %+v, %v", fresh, f) } diff --git a/modules/mesh-delivery-assigner/internal/assignment/digest.go b/modules/mesh-delivery-assigner/internal/assignment/digest.go index b22e05c6..a7166a1a 100644 --- a/modules/mesh-delivery-assigner/internal/assignment/digest.go +++ b/modules/mesh-delivery-assigner/internal/assignment/digest.go @@ -29,6 +29,8 @@ type placedForm struct { HasLayer bool `json:"has-layer,omitempty"` MeshValues map[string]any `json:"mesh-wide-values,omitempty"` Values map[string]any `json:"values,omitempty"` + Branch string `json:"branch,omitempty"` + Build string `json:"build,omitempty"` } type holdingForm struct { @@ -110,6 +112,8 @@ type changeForm struct { Whole *assignmentsForm `json:"whole,omitempty"` Decided *decidedForm `json:"decided,omitempty"` Keys *joiningForm `json:"keys,omitempty"` + Branch string `json:"branch,omitempty"` + Build string `json:"build,omitempty"` } func providerOf(p seat.Provider) providerForm { return providerForm{Node: p.Node, Module: p.Module} } @@ -150,7 +154,7 @@ func assignmentsOf(p seat.Assignments) assignmentsForm { } for _, m := range p.Modules { out.Modules = append(out.Modules, placedForm{Module: m.Module, Ports: m.Ports, Bindings: providersOf(m.Bindings), - MeshWide: m.MeshWide, HasMesh: m.HasMeshWide, Layer: m.Layer, HasLayer: m.HasLayer, MeshValues: m.MeshWideValues, Values: m.Values}) + MeshWide: m.MeshWide, HasMesh: m.HasMeshWide, Layer: m.Layer, HasLayer: m.HasLayer, MeshValues: m.MeshWideValues, Values: m.Values, Branch: m.Branch, Build: m.Build}) } for _, h := range p.Holds { out.Holds = append(out.Holds, holdingForm{Claim: h.Claim, Scope: h.Scope, Module: h.Module, Site: h.Site, Kind: h.Kind}) @@ -161,7 +165,7 @@ func assignmentsOf(p seat.Assignments) assignmentsForm { func changeOf(c seat.ProposalChange) changeForm { out := changeForm{Op: c.Op, Node: c.Node, Module: c.Module, Ports: c.Ports, Bindings: providersOf(c.Bindings), Provision: c.Provision, Claim: c.Claim, Scope: c.Scope, Kind: c.Kind, Site: c.Site, Field: c.Field, MeshWide: c.MeshWide, Layer: c.Layer, - MeshValues: c.MeshWideValues, Values: c.Values} + MeshValues: c.MeshWideValues, Values: c.Values, Branch: c.Branch, Build: c.Build} if c.Provider != nil { p := providerOf(*c.Provider) out.Provider = &p diff --git a/modules/mesh-delivery-assigner/internal/assignment/proposal.go b/modules/mesh-delivery-assigner/internal/assignment/proposal.go index b66cf164..1680e598 100644 --- a/modules/mesh-delivery-assigner/internal/assignment/proposal.go +++ b/modules/mesh-delivery-assigner/internal/assignment/proposal.go @@ -148,11 +148,19 @@ func LayChanges(current map[string]seat.Assignments, changes []seat.ProposalChan } continue case client.OpPlace: - m := seat.PlacedModule{Module: c.Module, Ports: c.Ports, Bindings: c.Bindings} + // A module's watched branch and its build are in the placement key, set by a proposal (design 47 line 1051, + // the Choices table's row): the change's when it names them, else kept from the module placed there. + m := seat.PlacedModule{Module: c.Module, Ports: c.Ports, Bindings: c.Bindings, Branch: c.Branch, Build: c.Build} if at >= 0 { o := p.Modules[at] m.MeshWide, m.HasMeshWide, m.Layer, m.HasLayer = o.MeshWide, o.HasMeshWide, o.Layer, o.HasLayer m.MeshWideValues, m.Values = o.MeshWideValues, o.Values + if m.Branch == "" { + m.Branch = o.Branch + } + if m.Build == "" { + m.Build = o.Build + } p.Modules[at] = m } else { p.Modules = append(p.Modules, m)