Keep the head last written per pull request in a bucket of the bridge's own instead of scanning FORGE, and test the bridge under its composed permissions (issue 1279, ADR 0443)
mesh/repo-check head ca667af562b0 merged into trunk 1325c8b452c8 beside mesh-controller#400@a0c971ce — pass: its merge-check.sh passed
mesh/merge-gate head ca667af562b0 merged into trunk 1325c8b452c8 — pass: group fix/1279-consumers-from-now composed: every machine composes with the change…
mesh/delivery delivered
mesh/delivery-group group fix/1279-consumers-from-now delivered: every member is delivered
mesh/repo-check head ca667af562b0 merged into trunk 1325c8b452c8 beside mesh-controller#400@a0c971ce — pass: its merge-check.sh passed
mesh/merge-gate head ca667af562b0 merged into trunk 1325c8b452c8 — pass: group fix/1279-consumers-from-now composed: every machine composes with the change…
mesh/delivery delivered
mesh/delivery-group group fix/1279-consumers-from-now delivered: every member is delivered
This commit is contained in:
@@ -3,50 +3,33 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
|
||||
// forgeStream is the bridge's own stream as the bus names it.
|
||||
const forgeStream = "FORGE"
|
||||
// headsBucket is the module's state bucket heads, `<module>_<name>` as the indexer's buckets are named: the head last
|
||||
// written on FORGE for each pull request (manifest state entry heads, novox/hq ADR 0443 §5).
|
||||
const headsBucket = delivery.ForgeBridgeModule + "_heads"
|
||||
|
||||
// heldOnForge reads the head FORGE last holds for each pull request: every pull and merged message the stream keeps, in
|
||||
// order, through an ordered consumer of no name that goes when this returns. It reads up to what is on the stream now.
|
||||
func heldOnForge(ctx context.Context, js jetstream.JetStream) (bridge.Held, error) {
|
||||
stream, err := js.Stream(ctx, forgeStream)
|
||||
// kvHeads is the bucket heads as the bridge's headStore.
|
||||
type kvHeads struct{ kv jetstream.KeyValue }
|
||||
|
||||
func (s kvHeads) Get(ctx context.Context, key string) (string, bool, error) {
|
||||
e, err := s.kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return "", false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the stream %s: %w", forgeStream, err)
|
||||
}
|
||||
cons, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{
|
||||
FilterSubjects: []string{forgeSubject("forge.*.*.pull"), forgeSubject("forge.*.*.merged")},
|
||||
DeliverPolicy: jetstream.DeliverAllPolicy,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", forgeStream, err)
|
||||
}
|
||||
held := bridge.Held{}
|
||||
for {
|
||||
batch, err := cons.FetchNoWait(256)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading %s: %w", forgeStream, err)
|
||||
}
|
||||
n := 0
|
||||
for m := range batch.Messages() {
|
||||
n++
|
||||
if f, err := delivery.DecodeForge(m.Data()); err == nil {
|
||||
held.Hear(f)
|
||||
}
|
||||
}
|
||||
if err := batch.Error(); err != nil && !errors.Is(err, jetstream.ErrNoMessages) {
|
||||
return nil, fmt.Errorf("reading %s: %w", forgeStream, err)
|
||||
}
|
||||
if n == 0 {
|
||||
return held, nil
|
||||
}
|
||||
return "", false, err
|
||||
}
|
||||
return string(e.Value()), true, nil
|
||||
}
|
||||
|
||||
func (s kvHeads) Put(ctx context.Context, key, head string) error {
|
||||
_, err := s.kv.PutString(ctx, key, head)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s kvHeads) Delete(ctx context.Context, key string) error { return s.kv.Delete(ctx, key) }
|
||||
|
||||
+69
-20
@@ -22,7 +22,14 @@ import (
|
||||
)
|
||||
|
||||
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream,
|
||||
// the controller's user and the bridge's own principal <node>.<module>.
|
||||
// the controller's user, and the bridge's own principal <node>.<module> with the permissions the controller composes for
|
||||
// it, and no others, so a permission the bridge lacks fails a test here and not on the mesh (issue 1279).
|
||||
//
|
||||
// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal
|
||||
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls and
|
||||
// gitea.gitea_commit_trees, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
|
||||
// WirePattern). The consumer's durable is <node>_<module> on EVENTS (nats.go consumerDurable). The controller's own
|
||||
// user keeps all rights, as it has.
|
||||
const busConf = `
|
||||
listen: 127.0.0.1:-1
|
||||
jetstream { store_dir: %q }
|
||||
@@ -31,7 +38,40 @@ accounts {
|
||||
jetstream: enabled
|
||||
users = [
|
||||
{ user: "ctl", password: "pw" }
|
||||
{ user: "n1.mesh-delivery-forge-bridge", password: "pw" }
|
||||
{ user: "n1.mesh-delivery-forge-bridge", password: "pw"
|
||||
permissions {
|
||||
publish { allow: [
|
||||
"$JS.ACK.EVENTS.n1_mesh-delivery-forge-bridge.>"
|
||||
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-forge-bridge_heads.>"
|
||||
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-forge-bridge_heads.>"
|
||||
"$JS.API.CONSUMER.INFO.EVENTS.n1_mesh-delivery-forge-bridge"
|
||||
"$JS.API.CONSUMER.MSG.NEXT.EVENTS.n1_mesh-delivery-forge-bridge"
|
||||
"$JS.API.DIRECT.GET.KV_mesh-delivery-forge-bridge_heads.>"
|
||||
"$JS.API.STREAM.INFO.KV_mesh-delivery-forge-bridge_heads"
|
||||
"$JS.FC.KV_mesh-delivery-forge-bridge_heads.>"
|
||||
"$KV.mesh-delivery-forge-bridge_heads.>"
|
||||
"mesh.mod.gitea.call.gitea_commit_trees.*.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_commit_trees.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_open_pulls.*.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.call.gitea_open_pulls.n1~mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.tool.gitea_commit_trees"
|
||||
"mesh.mod.gitea.tool.gitea_commit_trees.*"
|
||||
"mesh.mod.gitea.tool.gitea_open_pulls"
|
||||
"mesh.mod.gitea.tool.gitea_open_pulls.*"
|
||||
"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"
|
||||
] }
|
||||
subscribe { allow: [
|
||||
"_INBOX.n1.mesh-delivery-forge-bridge.>"
|
||||
"mesh.assignment.n1.mesh-delivery-forge-bridge"
|
||||
"mesh.mod.gitea.event.pull.closed"
|
||||
"mesh.mod.gitea.event.pull.merged"
|
||||
"mesh.mod.gitea.event.pull.updated"
|
||||
"mesh.mod.mesh-delivery-forge-bridge.call.>"
|
||||
"mesh.mod.mesh-delivery-forge-bridge.tool.>"
|
||||
] }
|
||||
allow_responses: true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -105,6 +145,10 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The bridge's state bucket, made by the controller from the manifest's state entry before the process starts.
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: headsBucket, History: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if prepare != nil {
|
||||
prepare(ctx, js)
|
||||
}
|
||||
@@ -163,12 +207,13 @@ func TestBridgeWritesPullMessages(t *testing.T) { // 1235/AC2
|
||||
}
|
||||
var info *jetstream.StreamInfo
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if info, err = forge.Info(ctx); err == nil && info.State.Msgs >= 1 {
|
||||
if info, err = forge.Info(ctx); err == nil && info.State.Msgs >= 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if info.State.Msgs != 1 {
|
||||
t.Fatalf("FORGE holds %d messages within 5 s, want 1", info.State.Msgs)
|
||||
// The open message the bridge writes at its start (ADR 0443 §5), and the event's message.
|
||||
if info.State.Msgs != 2 {
|
||||
t.Fatalf("FORGE holds %d messages within 5 s, want 2", info.State.Msgs)
|
||||
}
|
||||
raw, err := forge.GetLastMsgForSubject(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.pull")
|
||||
if err != nil {
|
||||
@@ -234,12 +279,13 @@ func TestBridgeWritesMergedMessage(t *testing.T) { // 1236/AC4
|
||||
}
|
||||
var info *jetstream.StreamInfo
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if info, err = forge.Info(ctx); err == nil && info.State.Msgs >= 1 {
|
||||
if info, err = forge.Info(ctx); err == nil && info.State.Msgs >= 2 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if info.State.Msgs != 1 {
|
||||
t.Fatalf("FORGE holds %d messages within 5 s, want 1", info.State.Msgs)
|
||||
// The open message the bridge writes at its start (ADR 0443 §5), and the event's message.
|
||||
if info.State.Msgs != 2 {
|
||||
t.Fatalf("FORGE holds %d messages within 5 s, want 2", info.State.Msgs)
|
||||
}
|
||||
raw, err := forge.GetLastMsgForSubject(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-delivery.merged")
|
||||
if err != nil {
|
||||
@@ -324,8 +370,8 @@ func TestBridgeExitsWhenTheOpenPullsCannotBeRead(t *testing.T) { // 1237/AC3
|
||||
}
|
||||
|
||||
// TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent starts the bridge whole against a fake forge with two
|
||||
// open pull requests, one of them with its head already on FORGE, and a bus whose EVENTS holds older pull events from
|
||||
// before its consumer was made. At start it writes the one pull message that differs and the open message naming both;
|
||||
// open pull requests, one of them with its head already kept in the bucket heads, and a bus whose EVENTS holds older pull events from
|
||||
// before its consumer was made, under the permissions the controller composes for the bridge. At start it writes the one pull message that differs and the open message naming both;
|
||||
// the bus connection closed, it reads the forge again and writes the open message alone, since every head is held now;
|
||||
// for the older events it writes nothing (ADR 0443 §1, §5).
|
||||
func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *testing.T) { // 1279/AC8
|
||||
@@ -348,9 +394,12 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
// What FORGE already holds: the head of mesh-delivery#12.
|
||||
raw, _ := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated})
|
||||
if _, err := js.Publish(ctx, pullSubject("mesh-delivery"), raw); err != nil {
|
||||
// What the bridge kept of an earlier write: the head of mesh-delivery#12.
|
||||
kv, err := js.KeyValue(ctx, headsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.PutString(ctx, "novox.mesh-delivery.12", head); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
@@ -388,12 +437,12 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes
|
||||
return len(got) == 2 && got[0] == bothOpen[0] && got[1] == bothOpen[1]
|
||||
}
|
||||
|
||||
// At start: the held head (message 1), the one that differs (2), the open message naming both (3).
|
||||
waitMsgs(3)
|
||||
if raw, err := forge.GetMsg(b.ctx, 2); err != nil || raw.Subject != pullSubject("mesh-catalog") {
|
||||
t.Fatalf("message 2 = %v %v, want mesh-catalog's pull message", raw, err)
|
||||
// At start: the one pull message that differs (1), the open message naming both (2).
|
||||
waitMsgs(2)
|
||||
if raw, err := forge.GetMsg(b.ctx, 1); err != nil || raw.Subject != pullSubject("mesh-catalog") {
|
||||
t.Fatalf("message 1 = %v %v, want mesh-catalog's pull message", raw, err)
|
||||
}
|
||||
if got := named(3); !same(got) {
|
||||
if got := named(2); !same(got) {
|
||||
t.Fatalf("the open message names %v, want %v", got, bothOpen)
|
||||
}
|
||||
|
||||
@@ -414,8 +463,8 @@ func TestBridgeReadsTheForgeAtStartAndAfterAReconnectAndHearsNoOlderEvent(t *tes
|
||||
if closed != 1 {
|
||||
t.Fatalf("%d connections of the bridge found", closed)
|
||||
}
|
||||
waitMsgs(4)
|
||||
if got := named(4); !same(got) {
|
||||
waitMsgs(3)
|
||||
if got := named(3); !same(got) {
|
||||
t.Fatalf("after the reconnect the open message names %v, want %v", got, bothOpen)
|
||||
}
|
||||
|
||||
|
||||
@@ -188,8 +188,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
call := callerOn(conn, cred.User)
|
||||
// No forge event is heard while the open pull requests are read and said (ADR 0443 §5): events and syncs take turns.
|
||||
var turn sync.Mutex
|
||||
held := func(ctx context.Context) (bridge.Held, error) { return heldOnForge(ctx, js) }
|
||||
said := func(ctx context.Context) error { return syncOpen(ctx, call, js, held, &turn, logger) }
|
||||
kv, err := js.KeyValue(ctx, headsBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the bucket %s: %w", headsBucket, err)
|
||||
}
|
||||
heads := kvHeads{kv}
|
||||
// Everything written on FORGE, an event's and a sync's, keeps the head in the bucket.
|
||||
rec := &recorder{pub: js, heads: heads, logger: logger}
|
||||
said := func(ctx context.Context) error { return syncOpen(ctx, call, rec, heads, &turn, logger) }
|
||||
if err := said(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -198,7 +204,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, eventsStream, err)
|
||||
}
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) { hear(&turn, func() { handle(ctx, m, js, treesVia(ctx, call), logger) }) },
|
||||
cc, err := cons.Consume(func(m jetstream.Msg) {
|
||||
hear(&turn, func() {
|
||||
// Bounded: the turn is held, and no sync or other event goes on without it.
|
||||
hctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
handle(hctx, m, rec, treesVia(hctx, call), logger)
|
||||
})
|
||||
},
|
||||
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -4,7 +4,13 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
@@ -13,8 +19,50 @@ import (
|
||||
// gitea_open_pulls: no arguments, answers {pulls: [...]}).
|
||||
const openPullsTool = "gitea.gitea_open_pulls"
|
||||
|
||||
// heldReader reads the head FORGE last holds for each pull request.
|
||||
type heldReader func(ctx context.Context) (bridge.Held, error)
|
||||
// headStore is the bucket heads: the head last written on FORGE for each pull request, by bridge.Track's key
|
||||
// (novox/hq ADR 0443 §5).
|
||||
type headStore interface {
|
||||
// Get is the head kept under the key; found is false when there is none.
|
||||
Get(ctx context.Context, key string) (head string, found bool, err error)
|
||||
Put(ctx context.Context, key, head string) error
|
||||
Delete(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
// recorder writes to FORGE and keeps in the bucket what it wrote: a pull message keeps its head, a closed or merged one
|
||||
// takes its pull request's key away. Every write of the program goes through it, an event's and a sync's alike.
|
||||
type recorder struct {
|
||||
pub publisher
|
||||
heads headStore
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
func (r *recorder) Publish(ctx context.Context, subject string, payload []byte, opts ...jetstream.PublishOpt) (*jetstream.PubAck, error) {
|
||||
ack, err := r.pub.Publish(ctx, subject, payload, opts...)
|
||||
if err != nil {
|
||||
return ack, err
|
||||
}
|
||||
f, derr := delivery.DecodeForge(payload)
|
||||
if derr != nil {
|
||||
return ack, nil
|
||||
}
|
||||
key, head, ok := bridge.Track(f)
|
||||
if !ok {
|
||||
return ack, nil
|
||||
}
|
||||
if head == "" {
|
||||
err = r.heads.Delete(ctx, key)
|
||||
} else {
|
||||
err = r.heads.Put(ctx, key, head)
|
||||
}
|
||||
if err != nil {
|
||||
return ack, fmt.Errorf("%s was written on FORGE but its head could not be kept in the bucket: %w", subject, err)
|
||||
}
|
||||
return ack, nil
|
||||
}
|
||||
|
||||
// syncDeadline bounds one sync: the call to the forge and every read and write of the bucket and of FORGE, because
|
||||
// the sync holds the turn that stops the bridge hearing any event.
|
||||
const syncDeadline = 30 * time.Second
|
||||
|
||||
// hear runs one reaction to a forge event while no sync of the open pull requests is between reading the forge and
|
||||
// writing the open message (novox/hq ADR 0443 §5): an event heard meanwhile waits, and is written after it, so a pull
|
||||
@@ -25,15 +73,17 @@ func hear(mu *sync.Mutex, react func()) {
|
||||
react()
|
||||
}
|
||||
|
||||
// syncOpen reads the open pull requests from the forge and writes a pull message for each whose head FORGE does not
|
||||
// hold, then one open message naming every one it read, those whose head did not differ among them: what makes the
|
||||
// indexer's candidates match the forge after a restart or a gap in the forge's events (ADR 0429 §13, ADR 0443 §5). It
|
||||
// holds mu from before it reads the forge until the open message is written, so no event is heard between. What FORGE
|
||||
// holds that cannot be read is logged, and every open pull request is written. An error names the call and the forge's
|
||||
// words.
|
||||
func syncOpen(ctx context.Context, call caller, pub publisher, held heldReader, mu *sync.Mutex, logger *log.Logger) error {
|
||||
// syncOpen reads the open pull requests from the forge and writes a pull message for each whose head differs from the
|
||||
// one kept in the bucket, then one open message naming every one it read, those whose head did not differ among them:
|
||||
// what makes the indexer's candidates match the forge after a restart or a gap in the forge's events (ADR 0429 §13,
|
||||
// ADR 0443 §5). pub keeps the bucket as it writes (recorder). It holds mu from before it reads the forge until the open
|
||||
// message is written, so no event is heard between, and never past syncDeadline. A bucket that cannot be read is logged
|
||||
// and every open pull request is written. An error names the call and the forge's words.
|
||||
func syncOpen(ctx context.Context, call caller, pub publisher, heads headStore, mu *sync.Mutex, logger *log.Logger) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
ctx, cancel := context.WithTimeout(ctx, syncDeadline)
|
||||
defer cancel()
|
||||
raw, err := call(ctx, openPullsTool, struct{}{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", openPullsTool, err)
|
||||
@@ -49,17 +99,30 @@ func syncOpen(ctx context.Context, call caller, pub publisher, held heldReader,
|
||||
for _, s := range skipped {
|
||||
logger.Printf("an open pull request is left out: %s", s)
|
||||
}
|
||||
if h, err := held(ctx); err != nil {
|
||||
logger.Printf("what FORGE holds could not be read, so every open pull request is written: %v", err)
|
||||
} else {
|
||||
msgs = h.Differing(msgs)
|
||||
var differing []bridge.Message
|
||||
for _, m := range msgs {
|
||||
f, err := delivery.DecodeForge(m.Data)
|
||||
if err != nil {
|
||||
differing = append(differing, m)
|
||||
continue
|
||||
}
|
||||
key, head, _ := bridge.Track(f)
|
||||
kept, found, err := heads.Get(ctx, key)
|
||||
if err != nil {
|
||||
logger.Printf("the heads kept could not be read, so every open pull request is written: %v", err)
|
||||
differing = slices.Clone(msgs)
|
||||
break
|
||||
}
|
||||
if !found || kept != head {
|
||||
differing = append(differing, m)
|
||||
}
|
||||
}
|
||||
for _, m := range append(msgs, open) {
|
||||
for _, m := range append(differing, open) {
|
||||
if _, err := pub.Publish(ctx, forgeSubject(m.Event), m.Data); err != nil {
|
||||
return fmt.Errorf("the open pull request message %s could not be written on FORGE: %w", m.Event, err)
|
||||
}
|
||||
}
|
||||
logger.Printf("%d pull messages and the open message written on FORGE", len(msgs))
|
||||
logger.Printf("%d pull messages and the open message written on FORGE", len(differing))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -14,8 +14,6 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-forge-bridge/internal/bridge"
|
||||
)
|
||||
|
||||
// openAnswer is gitea_open_pulls's answer as mesh-catalog modules/gitea/client.ts:369-381 builds its entries inside {pulls}.
|
||||
@@ -34,7 +32,7 @@ func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1,
|
||||
keys = append(keys, key)
|
||||
return openAnswer(), nil
|
||||
}
|
||||
if err := syncOpen(context.Background(), call, pub, nothingHeld, &sync.Mutex{}, log.New(&logs, "", 0)); err != nil {
|
||||
if err := runSync(call, pub, newHeads(), &sync.Mutex{}, log.New(&logs, "", 0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(keys) != 1 || keys[0] != "gitea.gitea_open_pulls" {
|
||||
@@ -52,22 +50,59 @@ func TestSyncOpenWritesOneUpdatedMessagePerOpenPull(t *testing.T) { // 1237/AC1,
|
||||
}
|
||||
}
|
||||
|
||||
func nothingHeld(context.Context) (bridge.Held, error) { return bridge.Held{}, nil }
|
||||
// memHeads is the bucket heads in memory.
|
||||
type memHeads struct {
|
||||
mu sync.Mutex
|
||||
vals map[string]string
|
||||
getErr error
|
||||
}
|
||||
|
||||
func newHeads(kv ...string) *memHeads {
|
||||
m := &memHeads{vals: map[string]string{}}
|
||||
for i := 0; i+1 < len(kv); i += 2 {
|
||||
m.vals[kv[i]] = kv[i+1]
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *memHeads) Get(_ context.Context, key string) (string, bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
v, ok := m.vals[key]
|
||||
return v, ok, m.getErr
|
||||
}
|
||||
func (m *memHeads) Put(_ context.Context, key, head string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.vals[key] = head
|
||||
return nil
|
||||
}
|
||||
func (m *memHeads) Delete(_ context.Context, key string) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.vals, key)
|
||||
return nil
|
||||
}
|
||||
func (m *memHeads) get(key string) string { m.mu.Lock(); defer m.mu.Unlock(); return m.vals[key] }
|
||||
|
||||
// runSync runs syncOpen with the recorder the program writes through.
|
||||
func runSync(call caller, pub publisher, heads headStore, mu *sync.Mutex, logger *log.Logger) error {
|
||||
return syncOpen(context.Background(), call, &recorder{pub: pub, heads: heads, logger: logger}, heads, mu, logger)
|
||||
}
|
||||
|
||||
// 1279/AC6: the sync writes a pull message only for a head that differs from what FORGE holds, then one open message
|
||||
// naming every open pull request into trunk it read, those whose head did not differ among them.
|
||||
func TestSyncOpenWritesPullsForNewHeadsThenOneOpenMessageNamingAll(t *testing.T) { // 1279/AC6
|
||||
pub := &fakePub{}
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) { return openAnswer(), nil }
|
||||
held := func(context.Context) (bridge.Held, error) {
|
||||
h := bridge.Held{}
|
||||
h.Hear(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated})
|
||||
return h, nil
|
||||
}
|
||||
heads := newHeads("novox.mesh-delivery.12", head)
|
||||
var logs bytes.Buffer
|
||||
if err := syncOpen(context.Background(), call, pub, held, &sync.Mutex{}, log.New(&logs, "", 0)); err != nil {
|
||||
if err := runSync(call, pub, heads, &sync.Mutex{}, log.New(&logs, "", 0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if heads.get("novox.mesh-catalog.300") != strings.Repeat("a", 40) || heads.get("novox.mesh-delivery.12") != head {
|
||||
t.Errorf("the bucket after the sync: %v, want the head written for mesh-catalog#300 kept beside the held one", heads.vals)
|
||||
}
|
||||
if len(pub.subjects) != 2 {
|
||||
t.Fatalf("%d messages written (%v), want the one pull message that differs and the open message", len(pub.subjects), pub.subjects)
|
||||
}
|
||||
@@ -84,21 +119,59 @@ func TestSyncOpenWritesPullsForNewHeadsThenOneOpenMessageNamingAll(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
// What FORGE holds cannot be read: every open pull request is written, as before, and the open message follows; the
|
||||
// The bucket cannot be read: every open pull request is written, as before, and the open message follows; the
|
||||
// indexer's writes are idempotent, so the fault costs messages and no state.
|
||||
func TestSyncOpenWritesAllWhenFORGECannotBeRead(t *testing.T) { // 1279/AC6
|
||||
func TestSyncOpenWritesAllWhenTheHeadsCannotBeRead(t *testing.T) { // 1279/AC6
|
||||
pub := &fakePub{}
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) { return openAnswer(), nil }
|
||||
held := func(context.Context) (bridge.Held, error) { return nil, errors.New("no consumer on FORGE") }
|
||||
heads := newHeads("novox.mesh-delivery.12", head)
|
||||
heads.getErr = errors.New("the bucket is gone")
|
||||
var logs bytes.Buffer
|
||||
if err := syncOpen(context.Background(), call, pub, held, &sync.Mutex{}, log.New(&logs, "", 0)); err != nil {
|
||||
if err := runSync(call, pub, heads, &sync.Mutex{}, log.New(&logs, "", 0)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(pub.subjects) != 3 || !strings.Contains(logs.String(), "no consumer on FORGE") {
|
||||
if len(pub.subjects) != 3 || !strings.Contains(logs.String(), "the bucket is gone") {
|
||||
t.Errorf("messages %v, log %q", pub.subjects, logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
// What the bridge writes on FORGE it keeps in the bucket: a head by its pull message, nothing by a closed or merged one.
|
||||
func TestWhatIsWrittenOnFORGEIsKeptInTheBucket(t *testing.T) { // 1279/AC6
|
||||
pub := &fakePub{}
|
||||
heads := newHeads()
|
||||
r := &recorder{pub: pub, heads: heads, logger: log.New(&bytes.Buffer{}, "", 0)}
|
||||
write := func(f delivery.Forge) {
|
||||
t.Helper()
|
||||
ev, _ := delivery.ForgeEvent(f.Repository, f.Kind)
|
||||
raw, err := delivery.EncodeForge(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := r.Publish(context.Background(), forgeSubject(ev), raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p := delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated}
|
||||
write(p)
|
||||
if heads.get("novox.mesh-delivery.12") != head {
|
||||
t.Fatalf("bucket %v", heads.vals)
|
||||
}
|
||||
p.Action = delivery.PullClosed
|
||||
write(p)
|
||||
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
|
||||
t.Errorf("a closed pull request is still kept: %v", heads.vals)
|
||||
}
|
||||
p.Action = delivery.PullUpdated
|
||||
write(p)
|
||||
write(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
|
||||
if _, kept := heads.vals["novox.mesh-delivery.12"]; kept {
|
||||
t.Errorf("a merged pull request is still kept: %v", heads.vals)
|
||||
}
|
||||
if len(pub.subjects) != 4 {
|
||||
t.Errorf("%d messages written", len(pub.subjects))
|
||||
}
|
||||
}
|
||||
|
||||
// 1279/AC6: no forge event is heard between reading the forge and writing the open message.
|
||||
func TestNoForgeEventIsHeardWhileTheOpenPullsAreSaid(t *testing.T) { // 1279/AC6
|
||||
pub := &lockedPub{}
|
||||
@@ -112,7 +185,7 @@ func TestNoForgeEventIsHeardWhileTheOpenPullsAreSaid(t *testing.T) { // 1279/AC6
|
||||
var mu sync.Mutex
|
||||
synced := make(chan error, 1)
|
||||
go func() {
|
||||
synced <- syncOpen(context.Background(), call, pub, nothingHeld, &mu, log.New(&bytes.Buffer{}, "", 0))
|
||||
synced <- runSync(call, pub, newHeads(), &mu, log.New(&bytes.Buffer{}, "", 0))
|
||||
}()
|
||||
<-reading
|
||||
// An event arrives while the forge is being read.
|
||||
@@ -158,7 +231,7 @@ func TestSyncOpenFailureNamesTheCallAndTheForgesWords(t *testing.T) { // 1237/AC
|
||||
call := func(context.Context, string, any) (json.RawMessage, error) {
|
||||
return nil, errors.New("the forge said 502")
|
||||
}
|
||||
err := syncOpen(context.Background(), call, pub, nothingHeld, &sync.Mutex{}, log.New(&bytes.Buffer{}, "", 0))
|
||||
err := runSync(call, pub, newHeads(), &sync.Mutex{}, log.New(&bytes.Buffer{}, "", 0))
|
||||
if err == nil || !strings.Contains(err.Error(), "gitea.gitea_open_pulls") || !strings.Contains(err.Error(), "the forge said 502") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
@@ -189,34 +190,18 @@ func OpenPulls(answer []byte) (msgs []Message, skipped []string, err error) {
|
||||
return msgs, skipped, nil
|
||||
}
|
||||
|
||||
// Held is the head FORGE last holds for each pull request, by `<owner>/<name>#<number>`: what the bridge compares an
|
||||
// open pull request's head with, so it writes a pull message only for a head that differs (novox/hq ADR 0443 §5).
|
||||
type Held map[string]string
|
||||
|
||||
func heldKey(repository string, number int) string { return fmt.Sprintf("%s#%d", repository, number) }
|
||||
|
||||
// Hear takes one message of FORGE into what is held: a pull message holds its head, a closed one or a merged message
|
||||
// holds nothing for its pull request. Other kinds change nothing.
|
||||
func (h Held) Hear(f delivery.Forge) {
|
||||
key := heldKey(f.Repository, f.Number)
|
||||
// Track says what a message the bridge wrote changes in the head it keeps for each pull request (the bucket heads,
|
||||
// novox/hq ADR 0443 §5): the key `<owner>.<name>.<number>`, and the head to keep under it, or "" to take the key away
|
||||
// (a closed pull request, a merged one). ok is false for a message that keeps no head.
|
||||
func Track(f delivery.Forge) (key, head string, ok bool) {
|
||||
key = strings.ReplaceAll(f.Repository, "/", ".") + "." + strconv.Itoa(f.Number)
|
||||
switch {
|
||||
case f.Kind == delivery.ForgePull && f.Action != delivery.PullClosed:
|
||||
h[key] = f.Head
|
||||
return key, f.Head, true
|
||||
case f.Kind == delivery.ForgePull || f.Kind == delivery.ForgeMerged:
|
||||
delete(h, key)
|
||||
return key, "", true
|
||||
}
|
||||
}
|
||||
|
||||
// Differing keeps the pull messages whose head FORGE does not hold, in their order.
|
||||
func (h Held) Differing(msgs []Message) []Message {
|
||||
var out []Message
|
||||
for _, m := range msgs {
|
||||
if f, err := delivery.DecodeForge(m.Data); err == nil && h[heldKey(f.Repository, f.Number)] == f.Head {
|
||||
continue
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// OpenMessage turns the answer of gitea_open_pulls into the open message: every open pull request into the trunk, by
|
||||
|
||||
@@ -350,31 +350,25 @@ func TestOpenMessageNamesEveryOpenPullIntoTheTrunk(t *testing.T) { // 1279/AC6
|
||||
}
|
||||
}
|
||||
|
||||
func TestOnlyAHeadFORGEDoesNotHoldIsWritten(t *testing.T) { // 1279/AC6
|
||||
h2 := strings.Repeat("a", 40)
|
||||
msgs, _, err := OpenPulls(openAnswer(openEntry("mesh-delivery", 12, "main", "feat/x", head), openEntry("mesh-catalog", 300, "main", "feat/x", h2)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
func TestTrackSaysWhichHeadOfWhichPullRequestWasWritten(t *testing.T) { // 1279/AC6
|
||||
pullMsg := func(action string) delivery.Forge {
|
||||
return delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: action}
|
||||
}
|
||||
held := Held{}
|
||||
held.Hear(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated})
|
||||
got := held.Differing(msgs)
|
||||
if len(got) != 1 || got[0].Event != "forge.novox.mesh-catalog.pull" {
|
||||
t.Fatalf("written: %+v, want only mesh-catalog#300", got)
|
||||
for _, a := range []string{delivery.PullOpened, delivery.PullUpdated} {
|
||||
if key, h, ok := Track(pullMsg(a)); key != "novox.mesh-delivery.12" || h != head || !ok {
|
||||
t.Errorf("%s: Track = %q %q %v, want the head kept under a bucket key", a, key, h, ok)
|
||||
}
|
||||
}
|
||||
// Another head for the same pull request differs; a closed or merged one is no longer held.
|
||||
held.Hear(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: h2, Action: delivery.PullUpdated})
|
||||
if got := held.Differing(msgs[:1]); len(got) != 1 {
|
||||
t.Errorf("a pull request held at another head was left out")
|
||||
// A closed pull request and a merged one are no longer kept: their key goes.
|
||||
if key, h, ok := Track(pullMsg(delivery.PullClosed)); key != "novox.mesh-delivery.12" || h != "" || !ok {
|
||||
t.Errorf("closed: Track = %q %q %v", key, h, ok)
|
||||
}
|
||||
held.Hear(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullUpdated})
|
||||
held.Hear(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullClosed})
|
||||
if got := held.Differing(msgs[:1]); len(got) != 1 {
|
||||
t.Errorf("a closed pull request is still held")
|
||||
merged := delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}}
|
||||
if key, h, ok := Track(merged); key != "novox.mesh-delivery.12" || h != "" || !ok {
|
||||
t.Errorf("merged: Track = %q %q %v", key, h, ok)
|
||||
}
|
||||
held.Hear(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Action: delivery.PullOpened})
|
||||
held.Hear(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-delivery", Branch: "feat/x", Number: 12, Head: head, Merge: "m", Root: "r", Trees: map[string]string{"": "t"}})
|
||||
if got := held.Differing(msgs[:1]); len(got) != 1 {
|
||||
t.Errorf("a merged pull request is still held")
|
||||
// The open message and the rest hold no head.
|
||||
if _, _, ok := Track(delivery.Forge{Kind: delivery.ForgeOpen}); ok {
|
||||
t.Error("an open message was tracked")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,11 @@ type stream struct {
|
||||
Events []string `json:"events"`
|
||||
}
|
||||
|
||||
type state struct {
|
||||
Name string `json:"name"`
|
||||
History int `json:"history"`
|
||||
}
|
||||
|
||||
type artifact struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
@@ -37,6 +42,7 @@ type manifest struct {
|
||||
Slug string `json:"slug"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Streams []stream `json:"streams"`
|
||||
State []state `json:"state"`
|
||||
Reads []string `json:"reads"`
|
||||
Invokes []string `json:"invokes"`
|
||||
OwnSecrets map[string]string `json:"own-secrets"`
|
||||
@@ -96,8 +102,12 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
|
||||
if m.SecretsOwner != "mesh-delivery-forge-bridge" {
|
||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||
}
|
||||
// No state, no bus, no seat, no tool (ADR 0437 §4).
|
||||
for _, k := range []string{"state", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
|
||||
// One bucket: the head last written on FORGE for each pull request, so a sync writes only what differs (ADR 0443 §5).
|
||||
if !reflect.DeepEqual(m.State, []state{{Name: "heads", History: 1}}) { // 1279/AC6
|
||||
t.Errorf("state = %+v, want the one bucket heads", m.State)
|
||||
}
|
||||
// No other state, no bus, no seat, no tool (ADR 0437 §4).
|
||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats"} {
|
||||
if _, ok := top[k]; ok {
|
||||
t.Errorf("the manifest declares %q: the bridge has none", k)
|
||||
}
|
||||
|
||||
@@ -10,6 +10,9 @@
|
||||
"streams": [
|
||||
{ "name": "forge", "events": ["forge.**"] }
|
||||
],
|
||||
"state": [
|
||||
{ "name": "heads", "history": 1 }
|
||||
],
|
||||
"reads": [
|
||||
"mesh-delivery-gate.merges",
|
||||
"mesh-delivery-gate.owed"
|
||||
|
||||
Reference in New Issue
Block a user