novox/mesh-controller
13
0
Fork 0
Code Issues Pull Requests 4 Releases Wiki Activity

Merge branch 'feat/one-group-one-plan' into g/977

Browse Source
This commit is contained in:
jschoubben
2026-10-17 21:37:37 +02:00
parent 07b050790e 66dc501fc4
commit 64415f8984
438 changed files with 22014 additions and 19825 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/link/advisories_integration_test.go
+59
View File
@@ -0,0 +1,59 @@
//go:build integration
package link
import (
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS_REFUSING")
if url == "" {
t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>")
}
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
started := time.Now()
_, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second)
if err == nil || !strings.Contains(err.Error(), "the bus refused") {
t.Fatalf("answered %v", err)
}
if took := time.Since(started); took > 3*time.Second {
t.Fatalf("a refusal took %s to be known", took)
}
}
func TestNatsARefusalReachesItsWaiter(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor")
defer stop()
other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop")
defer stopOther()
conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q",
nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor"))
select {
case <-mine:
case <-time.After(time.Second):
t.Fatal("the waiter was not told")
}
select {
case <-other:
t.Fatal("another subject's waiter was told")
default:
}
}
internal/link/advisories_test.go
-51
View File
@@ -1,15 +1,7 @@
package link
import (
"fmt"
"os"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// **A reader's own consumer gone is not a consumer lost**: every bucket watch and stream read-back
@@ -45,48 +37,5 @@ func TestOnlyTheMeshsOwnConsumersAreSaidLost(t *testing.T) {
// **A question the bus refuses is answered as refused at once**, not after its timeout: against a
// server whose only user may not publish where it asks.
func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS_REFUSING")
if url == "" {
t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>")
}
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
started := time.Now()
_, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second)
if err == nil || !strings.Contains(err.Error(), "the bus refused") {
t.Fatalf("answered %v", err)
}
if took := time.Since(started); took > 3*time.Second {
t.Fatalf("a refusal took %s to be known", took)
}
}
// The refusal reaches whoever waits on that subject, and only them.
func TestNatsARefusalReachesItsWaiter(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor")
defer stop()
other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop")
defer stopOther()
conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q",
nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor"))
select {
case <-mine:
case <-time.After(time.Second):
t.Fatal("the waiter was not told")
}
select {
case <-other:
t.Fatal("another subject's waiter was told")
default:
}
}
internal/link/builds_nats_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/bus_integration_test.go
+124
View File
@@ -0,0 +1,124 @@
//go:build integration
package link
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) {
url := testbus.URL(t)
f := loadFixture(t, "events/module-event.json")
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
js, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := js.AddStream(&nats.StreamConfig{
Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"},
}); err != nil && err != nats.ErrStreamNameAlreadyInUse {
t.Fatal(err)
}
bus := OverNATS{JS: js}
body, _ := json.Marshal(f.Given.Body)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := EmitEvent(ctx, bus, f.Given.Key, f.Given.Module, f.Given.Node, f.Given.Body); err != nil {
t.Fatal(err)
}
// Read back from the stream, not from the thing that wrote it.
raw, err := js.GetLastMsg("EVENTS", f.Wire.Subject)
if err != nil {
t.Fatalf("nothing landed on %s, which the fixture names: %v", f.Wire.Subject, err)
}
for _, h := range f.Wire.RequiredHeaders {
if raw.Header.Get(h) == "" {
t.Errorf("%s is not set on the wire, and the fixture requires it", h)
}
}
if got := raw.Header.Get("x-source"); got != f.Given.Module {
t.Errorf("x-source is %q; the subject says %q", got, f.Given.Module)
}
if string(raw.Data) != string(body) {
t.Errorf("the payload is %s, expected the body alone: %s", raw.Data, body)
}
// The envelope must not also be nested inside the payload.
var nested map[string]any
if json.Unmarshal(raw.Data, &nested) == nil {
if _, has := nested["key"]; has {
t.Error("the payload carries the envelope's own fields, which the fixture refuses")
}
}
}
func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
bus := OverNATS{Conn: conn}
began := time.Now()
_, err = bus.AskTool(context.Background(), "nobody", "home", nil, 30*time.Second)
if err == nil {
t.Fatal("asking a tool nothing serves succeeded")
}
if took := time.Since(began); took > 2*time.Second {
t.Errorf("took %v to say nothing serves it; the caller waited out the timeout", took)
}
if !strings.Contains(err.Error(), "nothing serves") {
t.Errorf("the refusal does not say nobody is there: %v", err)
}
// And it is ErrNothingServes, through Ask too, so an asker can wait out a tool runner re-serving and
// never a refusal (novox/hq issue 486).
if !errors.Is(err, ErrNothingServes) {
t.Errorf("nothing serving the tool is not ErrNothingServes: %v", err)
}
if _, err := Ask(context.Background(), bus, "nobody", "home", nil, 30*time.Second); !errors.Is(err, ErrNothingServes) ||
!strings.Contains(err.Error(), "nothing serves nobody.home") {
t.Errorf("Ask does not say ErrNothingServes in its words: %v", err)
}
}
func TestAskingAServedToolAnswers(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
sub, err := conn.Subscribe(ToolSubject("shop", "price"), func(m *nats.Msg) {
_ = m.Respond([]byte(`{"total":12}`))
})
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe()
got, err := OverNATS{Conn: conn}.AskTool(context.Background(), "shop", "price",
[]byte(`{"qty":4}`), 5*time.Second)
if err != nil {
t.Fatal(err)
}
if string(got) != `{"total":12}` {
t.Fatalf("the answer came back as %s", got)
}
}
internal/link/bus_test.go
-115
View File
@@ -1,16 +1,7 @@
package link
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// **Both implementations, one fixture.** The point of the seam is not that the transport can be
@@ -20,57 +11,6 @@ import (
// Against a real server, because what the fixture pins is what reaches the wire:
//
// go test ./internal/link/ -run TestTheNatsBus (each test on a bus of its own: internal/testbus)
func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) {
url := testbus.URL(t)
f := loadFixture(t, "events/module-event.json")
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
js, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := js.AddStream(&nats.StreamConfig{
Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"},
}); err != nil && err != nats.ErrStreamNameAlreadyInUse {
t.Fatal(err)
}
bus := OverNATS{JS: js}
body, _ := json.Marshal(f.Given.Body)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := EmitEvent(ctx, bus, f.Given.Key, f.Given.Module, f.Given.Node, f.Given.Body); err != nil {
t.Fatal(err)
}
// Read back from the stream, not from the thing that wrote it.
raw, err := js.GetLastMsg("EVENTS", f.Wire.Subject)
if err != nil {
t.Fatalf("nothing landed on %s, which the fixture names: %v", f.Wire.Subject, err)
}
for _, h := range f.Wire.RequiredHeaders {
if raw.Header.Get(h) == "" {
t.Errorf("%s is not set on the wire, and the fixture requires it", h)
}
}
if got := raw.Header.Get("x-source"); got != f.Given.Module {
t.Errorf("x-source is %q; the subject says %q", got, f.Given.Module)
}
if string(raw.Data) != string(body) {
t.Errorf("the payload is %s, expected the body alone: %s", raw.Data, body)
}
// The envelope must not also be nested inside the payload.
var nested map[string]any
if json.Unmarshal(raw.Data, &nested) == nil {
if _, has := nested["key"]; has {
t.Error("the payload carries the envelope's own fields, which the fixture refuses")
}
}
}
// The subject a declaration lands on is one node's, and nothing else's — the state shape.
func TestADeclarationIsAddressedToOneNode(t *testing.T) {
@@ -96,60 +36,5 @@ func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) {
// A tool nobody serves says so at once. The difference between "that module is down" and "that
// tool is slow" is the first thing a person asking wants, and waiting out the timeout to say it
// is how a fast answer becomes a slow non-answer.
func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
bus := OverNATS{Conn: conn}
began := time.Now()
_, err = bus.AskTool(context.Background(), "nobody", "home", nil, 30*time.Second)
if err == nil {
t.Fatal("asking a tool nothing serves succeeded")
}
if took := time.Since(began); took > 2*time.Second {
t.Errorf("took %v to say nothing serves it; the caller waited out the timeout", took)
}
if !strings.Contains(err.Error(), "nothing serves") {
t.Errorf("the refusal does not say nobody is there: %v", err)
}
// And it is ErrNothingServes, through Ask too, so an asker can wait out a tool runner re-serving and
// never a refusal (novox/hq issue 486).
if !errors.Is(err, ErrNothingServes) {
t.Errorf("nothing serving the tool is not ErrNothingServes: %v", err)
}
if _, err := Ask(context.Background(), bus, "nobody", "home", nil, 30*time.Second); !errors.Is(err, ErrNothingServes) ||
!strings.Contains(err.Error(), "nothing serves nobody.home") {
t.Errorf("Ask does not say ErrNothingServes in its words: %v", err)
}
}
// And a served tool answers, with no reply queue to declare and no correlation to check.
func TestAskingAServedToolAnswers(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
sub, err := conn.Subscribe(ToolSubject("shop", "price"), func(m *nats.Msg) {
_ = m.Respond([]byte(`{"total":12}`))
})
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe()
got, err := OverNATS{Conn: conn}.AskTool(context.Background(), "shop", "price",
[]byte(`{"qty":4}`), 5*time.Second)
if err != nil {
t.Fatal(err)
}
if string(got) != `{"total":12}` {
t.Fatalf("the answer came back as %s", got)
}
}
internal/link/busfaults_integration_test.go
+96
View File
@@ -0,0 +1,96 @@
//go:build integration
package link
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
func aBusWithEvents(t *testing.T) *Server {
t.Helper()
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
return ConnectNats(js, nil, nil)
}
func TestNatsBusFaultsAreOnePerUserAndKindEachClosureCountedOnce(t *testing.T) {
s := aBusWithEvents(t)
now := time.Now().UTC()
slow := BusClosure{User: "laptop.claude-code", Name: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)",
Opened: now.Add(-20 * time.Minute), Closed: now.Add(-2 * time.Minute), Pending: 334000}
sayClosure(t, s, BusSlowConsumerEvent, slow)
sayClosure(t, s, BusSlowConsumerEvent, slow) // said again: the module started again
again := slow
again.CID, again.Opened, again.Closed = 31, now.Add(-90*time.Second), now.Add(-time.Minute)
sayClosure(t, s, BusSlowConsumerEvent, again)
sayClosure(t, s, BusConnectionClosedEvent, BusClosure{User: "anchor.desk", CID: 7,
Reason: "Maximum Message Payload Exceeded", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-30 * time.Second)})
// Older than the hour: S9 no longer says it.
sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "anchor.media", CID: 3, Reason: "Slow Consumer (Pending Bytes)",
Opened: now.Add(-3 * time.Hour), Closed: now.Add(-2 * time.Hour)})
got, err := s.BusFaults(t.Context(), now.Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
if len(got) != 2 {
t.Fatalf("read %d advisories, want one for each user within the hour: %+v", len(got), got)
}
for _, a := range got {
switch a.User {
case "laptop.claude-code":
if a.Kind != AdvisorySlowConsumer || a.ID != "laptop.claude-code" || a.Count != 2 ||
!a.Last.Equal(again.Closed) || !a.First.Equal(slow.Closed) || !strings.Contains(a.Said, "connection 31") {
t.Errorf("the slow consumer read as %+v: two closures, the one said twice counted once", a)
}
case "anchor.desk":
if a.Kind != AdvisoryConnectionClosed || a.Count != 1 || a.Reason != "Maximum Message Payload Exceeded" {
t.Errorf("the closed connection read as %+v", a)
}
default:
t.Errorf("an advisory for %s", a.User)
}
}
}
func TestNatsBusFaultsReadNothingWhenTheBusModuleSaidNothing(t *testing.T) {
s := aBusWithEvents(t)
started := time.Now()
got, err := s.BusFaults(t.Context(), time.Now().Add(-time.Hour))
if err != nil || len(got) != 0 {
t.Fatalf("read %v, %v from a stream the bus module never wrote", got, err)
}
if took := time.Since(started); took > time.Second {
t.Fatalf("reading nothing took %s: a tick must not wait on an empty stream", took)
}
}
func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) {
s := aBusWithEvents(t)
was := MaxBusEventsRead
MaxBusEventsRead = 2
t.Cleanup(func() { MaxBusEventsRead = was })
now := time.Now().UTC()
for i := range 3 {
sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "laptop.claude-code", CID: uint64(10 + i),
Reason: "Slow Consumer (Pending Bytes)", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-time.Duration(3-i) * time.Minute)})
}
got, err := s.BusFaults(t.Context(), now.Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].Count != 2 || !strings.Contains(got[0].Said, "at least: the read stopped at 2") {
t.Fatalf("a read past its bound read as %+v", got)
}
}
internal/link/busfaults_test.go
-87
View File
@@ -2,12 +2,8 @@ package link
import (
"encoding/json"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
// What the bus module says of its users' connections, read back as advisories (novox/hq ADR 0357, issue
@@ -15,19 +11,6 @@ import (
//
// go test ./internal/link/ -run TestNatsBusFaults (each test on a bus of its own: internal/testbus)
func aBusWithEvents(t *testing.T) *Server {
t.Helper()
js, err := broker.Dial(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
return ConnectNats(js, nil, nil)
}
func sayClosure(t *testing.T, s *Server, event string, c BusClosure) {
t.Helper()
body, _ := json.Marshal(c)
@@ -36,58 +19,6 @@ func sayClosure(t *testing.T, s *Server, event string, c BusClosure) {
}
}
func TestNatsBusFaultsAreOnePerUserAndKindEachClosureCountedOnce(t *testing.T) {
s := aBusWithEvents(t)
now := time.Now().UTC()
slow := BusClosure{User: "laptop.claude-code", Name: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)",
Opened: now.Add(-20 * time.Minute), Closed: now.Add(-2 * time.Minute), Pending: 334000}
sayClosure(t, s, BusSlowConsumerEvent, slow)
sayClosure(t, s, BusSlowConsumerEvent, slow) // said again: the module started again
again := slow
again.CID, again.Opened, again.Closed = 31, now.Add(-90*time.Second), now.Add(-time.Minute)
sayClosure(t, s, BusSlowConsumerEvent, again)
sayClosure(t, s, BusConnectionClosedEvent, BusClosure{User: "anchor.desk", CID: 7,
Reason: "Maximum Message Payload Exceeded", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-30 * time.Second)})
// Older than the hour: S9 no longer says it.
sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "anchor.media", CID: 3, Reason: "Slow Consumer (Pending Bytes)",
Opened: now.Add(-3 * time.Hour), Closed: now.Add(-2 * time.Hour)})
got, err := s.BusFaults(t.Context(), now.Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
if len(got) != 2 {
t.Fatalf("read %d advisories, want one for each user within the hour: %+v", len(got), got)
}
for _, a := range got {
switch a.User {
case "laptop.claude-code":
if a.Kind != AdvisorySlowConsumer || a.ID != "laptop.claude-code" || a.Count != 2 ||
!a.Last.Equal(again.Closed) || !a.First.Equal(slow.Closed) || !strings.Contains(a.Said, "connection 31") {
t.Errorf("the slow consumer read as %+v: two closures, the one said twice counted once", a)
}
case "anchor.desk":
if a.Kind != AdvisoryConnectionClosed || a.Count != 1 || a.Reason != "Maximum Message Payload Exceeded" {
t.Errorf("the closed connection read as %+v", a)
}
default:
t.Errorf("an advisory for %s", a.User)
}
}
}
func TestNatsBusFaultsReadNothingWhenTheBusModuleSaidNothing(t *testing.T) {
s := aBusWithEvents(t)
started := time.Now()
got, err := s.BusFaults(t.Context(), time.Now().Add(-time.Hour))
if err != nil || len(got) != 0 {
t.Fatalf("read %v, %v from a stream the bus module never wrote", got, err)
}
if took := time.Since(started); took > time.Second {
t.Fatalf("reading nothing took %s: a tick must not wait on an empty stream", took)
}
}
func TestTheControllersOwnSlowConsumerAndItsClosedConnectionAreOneAdvisory(t *testing.T) {
at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC)
own := []Advisory{{Kind: AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind", First: at, Last: at, Count: 1}}
@@ -101,21 +32,3 @@ func TestTheControllersOwnSlowConsumerAndItsClosedConnectionAreOneAdvisory(t *te
// **A busy hour is read up to a bound, and says it was cut** (review of #326): a storm of closures must not
// spend a watchdog tick, and a count cut short is a floor, said as one.
func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) {
s := aBusWithEvents(t)
was := MaxBusEventsRead
MaxBusEventsRead = 2
t.Cleanup(func() { MaxBusEventsRead = was })
now := time.Now().UTC()
for i := range 3 {
sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "laptop.claude-code", CID: uint64(10 + i),
Reason: "Slow Consumer (Pending Bytes)", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-time.Duration(3-i) * time.Minute)})
}
got, err := s.BusFaults(t.Context(), now.Add(-time.Hour))
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0].Count != 2 || !strings.Contains(got[0].Said, "at least: the read stopped at 2") {
t.Fatalf("a read past its bound read as %+v", got)
}
}
internal/link/calls_integration_test.go
+38
View File
@@ -0,0 +1,38 @@
//go:build integration
package link
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
func TestAHandlerThatPanicsAnswersAnError(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
stop, err := OverNATS{Conn: conn}.ServeSeatTools("panicky", map[string]ToolHandler{
"boom": func(context.Context, json.RawMessage) (any, error) { panic("nil map") },
"fine": func(context.Context, json.RawMessage) (any, error) { return "ok", nil },
}, nil)
if err != nil {
t.Fatal(err)
}
defer stop()
answer, err := AskMeshSeatTool(context.Background(), conn, "panicky", "boom", map[string]any{}, 5*time.Second)
if err != nil || !strings.Contains(answer.Error, "failed inside the controller") {
t.Fatalf("a panic answered %+v (%v)", answer, err)
}
answer, err = AskMeshSeatTool(context.Background(), conn, "panicky", "fine", map[string]any{}, 5*time.Second)
if err != nil || answer.Error != "" {
t.Fatalf("the call after a panic answered %+v (%v)", answer, err)
}
}
internal/link/calls_kept_integration_test.go
+219
View File
@@ -0,0 +1,219 @@
//go:build integration
package link
import (
"bytes"
"context"
"encoding/json"
"log"
"strings"
"testing"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
func callsBucket(t *testing.T) *BusCalls {
t.Helper()
js := aBus(t)
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
_ = api.DeleteKeyValue(t.Context(), broker.CallsBucket)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
keeper, err := CallsOnTheBus(t.Context(), js.Conn())
if err != nil {
t.Fatal(err)
}
return keeper
}
func TestNatsACallsOutcomeOutlivesItsController(t *testing.T) {
keeper := callsBucket(t)
first := NewCallLog()
if err := first.Durably(t.Context(), keeper, "controller@one", nil); err != nil {
t.Fatal(err)
}
a := newAnswers(t)
first.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor","values":"secret"}`),
"_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1",
func(context.Context, json.RawMessage) (any, error) { return "anchor told", nil }, a.respond, nil)
recent, _ := first.Recent()
finished := waitKept(t, keeper, recent[0].ID, CallAnswered)
// A second call, still running when its controller stops.
left := first.begin("mesh-controller", "plans", json.RawMessage(`{}`), "")
waitKept(t, keeper, left.ID, CallRunning)
var said bytes.Buffer
second := NewCallLog()
if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil {
t.Fatal(err)
}
c, found, err := second.Get(finished.ID)
if err != nil || !found {
t.Fatalf("the next controller cannot read %s: %v %v", finished.ID, found, err)
}
if !strings.Contains(string(c.Answer), "anchor told") || c.Caller != "node-tools.g14" || c.Holder != "controller@one" {
t.Fatalf("kept %+v", c)
}
if strings.Contains(string(c.Args), "secret") {
t.Fatalf("a setting was kept: %s", c.Args)
}
abandoned, _, _ := second.Get(left.ID)
if abandoned.State != CallAbandoned || !strings.Contains(said.String(), left.ID) {
t.Fatalf("a call left running reads %q, and was said: %q", abandoned.State, said.String())
}
listed, err := second.Recent()
if err != nil || len(listed) != 2 {
t.Fatalf("the next controller lists %d calls: %v", len(listed), err)
}
// Its own running calls are not its predecessor's: a controller starting again under the same
// name leaves them alone.
mine := second.begin("mesh-controller", "status", nil, "")
waitKept(t, keeper, mine.ID, CallRunning)
if err := second.Durably(t.Context(), keeper, "controller@two", nil); err != nil {
t.Fatal(err)
}
if c, _, _ := keeper.Kept(t.Context(), mine.ID); c.State != CallRunning {
t.Fatalf("a controller marked its own running call %q", c.State)
}
}
func TestNatsAFinishRacingTheSweepsSnapshotReadsFinished(t *testing.T) {
keeper := callsBucket(t)
running := Call{ID: "call-711-1", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"}
if err := keeper.Keep(t.Context(), running); err != nil {
t.Fatal(err)
}
waitKept(t, keeper, running.ID, CallRunning)
finished := running
finished.State = CallAnswered
finished.Answer = json.RawMessage(`"anchor told"`)
wrapped := racingRecent{CallKeeper: keeper, thenLand: func() {
if err := keeper.Keep(context.Background(), finished); err != nil {
t.Error(err)
}
waitKept(t, keeper, running.ID, CallAnswered)
}}
var said bytes.Buffer
second := NewCallLog()
if err := second.Durably(t.Context(), wrapped, "controller@two", log.New(&said, "", 0)); err != nil {
t.Fatal(err)
}
got, found, err := second.Get(running.ID)
if err != nil || !found || got.State != CallAnswered || !strings.Contains(string(got.Answer), "anchor told") {
t.Fatalf("its finish did not stand: %+v %v %v", got, found, err)
}
if strings.Contains(said.String(), running.ID) {
t.Errorf("a finished call was said abandoned: %q", said.String())
}
}
func TestNatsARunningCallWithNoFinishIsStillAbandoned(t *testing.T) {
keeper := callsBucket(t)
running := Call{ID: "call-711-2", Seat: "mesh-controller", Verb: "plans", State: CallRunning, Holder: "controller@one"}
if err := keeper.Keep(t.Context(), running); err != nil {
t.Fatal(err)
}
waitKept(t, keeper, running.ID, CallRunning)
var said bytes.Buffer
second := NewCallLog()
if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil {
t.Fatal(err)
}
got, found, err := second.Get(running.ID)
if err != nil || !found || got.State != CallAbandoned || !strings.Contains(said.String(), running.ID) {
t.Fatalf("got %+v %v %v; said %q", got, found, err, said.String())
}
}
func TestNatsAFinishRacingTheSweepsWriteWins(t *testing.T) {
keeper := callsBucket(t)
running := Call{ID: "call-711-3", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"}
if err := keeper.Keep(t.Context(), running); err != nil {
t.Fatal(err)
}
waitKept(t, keeper, running.ID, CallRunning)
finished := running
finished.State = CallFinishedAfter
finished.Answer = json.RawMessage(`"anchor told"`)
landed := false
racey := &BusCalls{kv: delayedKV{KeyValue: keeper.kv, before: func() {
if landed {
return
}
landed = true
if err := keeper.Keep(context.Background(), finished); err != nil {
t.Error(err)
}
waitKept(t, keeper, running.ID, CallFinishedAfter)
}}}
var said bytes.Buffer
second := NewCallLog()
if err := second.Durably(t.Context(), racey, "controller@two", log.New(&said, "", 0)); err != nil {
t.Fatal(err)
}
got, found, err := second.Get(running.ID)
if err != nil || !found || got.State != CallFinishedAfter || !strings.Contains(string(got.Answer), "anchor told") {
t.Fatalf("the finish did not win the race: %+v %v %v", got, found, err)
}
if strings.Contains(said.String(), running.ID) {
t.Errorf("a finished call was said abandoned: %q", said.String())
}
}
func TestNatsAMarkAbandonedGivesUpAfterTenTries(t *testing.T) {
keeper := callsBucket(t)
running := Call{ID: "call-711-4", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"}
if err := keeper.Keep(t.Context(), running); err != nil {
t.Fatal(err)
}
waitKept(t, keeper, running.ID, CallRunning)
tries := 0
stuck := &BusCalls{kv: alwaysMismatchKV{KeyValue: keeper.kv, tries: &tries}}
_, err := stuck.MarkAbandoned(t.Context(), running.ID)
if err == nil || !strings.Contains(err.Error(), running.ID) {
t.Fatalf("expected an error naming %s, got %v", running.ID, err)
}
if tries != markAbandonedTries {
t.Fatalf("tried %d times, want %d", tries, markAbandonedTries)
}
}
func TestNatsTheCallsBucketIsBounded(t *testing.T) {
callsBucket(t)
js := aBus(t)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
info, err := js.Context().StreamInfo("KV_" + broker.CallsBucket)
if err != nil {
t.Fatal(err)
}
if info.Config.MaxMsgs != 2*broker.KeptCallsDurably || info.Config.MaxAge != broker.CallsKeptFor {
t.Fatalf("kept %d messages for %s", info.Config.MaxMsgs, info.Config.MaxAge)
}
}
func TestNatsAnAnswerLargerThanTheBoundIsCut(t *testing.T) {
keeper := callsBucket(t)
big, _ := json.Marshal(map[string]string{"result": strings.Repeat("x", broker.CallAnswerBytes+10)})
c := Call{ID: "call-1-1", Seat: "mesh-controller", Verb: "plan", State: CallAnswered, Answer: big}
if err := keeper.Keep(t.Context(), c); err != nil {
t.Fatal(err)
}
got, found, err := keeper.Kept(t.Context(), "call-1-1")
if err != nil || !found || !strings.Contains(string(got.Answer), "the first") {
t.Fatalf("%v %v %.200s", found, err, got.Answer)
}
}
internal/link/calls_kept_test.go
-203
View File
@@ -1,40 +1,16 @@
package link
import (
"bytes"
"context"
"encoding/json"
"log"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
)
// The calls bucket against a real server (novox/hq to-be 45 §6): whether a call's outcome is
// readable by id from a controller that did not serve it is a claim about what the bus keeps.
func callsBucket(t *testing.T) *BusCalls {
t.Helper()
js := aBus(t)
api, err := jetstream.New(js.Conn())
if err != nil {
t.Fatal(err)
}
_ = api.DeleteKeyValue(t.Context(), broker.CallsBucket)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
keeper, err := CallsOnTheBus(t.Context(), js.Conn())
if err != nil {
t.Fatal(err)
}
return keeper
}
// waitKept waits until the keeper holds a call in a state, the writes being queued.
func waitKept(t *testing.T, keeper CallKeeper, id, state string) Call {
t.Helper()
@@ -54,56 +30,6 @@ func waitKept(t *testing.T, keeper CallKeeper, id, state string) Call {
// **A controller restart keeps every call's outcome** (to-be 45 Phase 0): a call one controller
// answered is read whole by id from the next, and a call it left running is said abandoned rather
// than running for ever.
func TestNatsACallsOutcomeOutlivesItsController(t *testing.T) {
keeper := callsBucket(t)
first := NewCallLog()
if err := first.Durably(t.Context(), keeper, "controller@one", nil); err != nil {
t.Fatal(err)
}
a := newAnswers(t)
first.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor","values":"secret"}`),
"_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1",
func(context.Context, json.RawMessage) (any, error) { return "anchor told", nil }, a.respond, nil)
recent, _ := first.Recent()
finished := waitKept(t, keeper, recent[0].ID, CallAnswered)
// A second call, still running when its controller stops.
left := first.begin("mesh-controller", "plans", json.RawMessage(`{}`), "")
waitKept(t, keeper, left.ID, CallRunning)
var said bytes.Buffer
second := NewCallLog()
if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil {
t.Fatal(err)
}
c, found, err := second.Get(finished.ID)
if err != nil || !found {
t.Fatalf("the next controller cannot read %s: %v %v", finished.ID, found, err)
}
if !strings.Contains(string(c.Answer), "anchor told") || c.Caller != "node-tools.g14" || c.Holder != "controller@one" {
t.Fatalf("kept %+v", c)
}
if strings.Contains(string(c.Args), "secret") {
t.Fatalf("a setting was kept: %s", c.Args)
}
abandoned, _, _ := second.Get(left.ID)
if abandoned.State != CallAbandoned || !strings.Contains(said.String(), left.ID) {
t.Fatalf("a call left running reads %q, and was said: %q", abandoned.State, said.String())
}
listed, err := second.Recent()
if err != nil || len(listed) != 2 {
t.Fatalf("the next controller lists %d calls: %v", len(listed), err)
}
// Its own running calls are not its predecessor's: a controller starting again under the same
// name leaves them alone.
mine := second.begin("mesh-controller", "status", nil, "")
waitKept(t, keeper, mine.ID, CallRunning)
if err := second.Durably(t.Context(), keeper, "controller@two", nil); err != nil {
t.Fatal(err)
}
if c, _, _ := keeper.Kept(t.Context(), mine.ID); c.State != CallRunning {
t.Fatalf("a controller marked its own running call %q", c.State)
}
}
// racingRecent wraps a CallKeeper so a finish lands right after its Recent() snapshot is taken, as
// if that finish raced the moment the snapshot was read (novox/hq issue 711): the snapshot still
@@ -125,58 +51,9 @@ func (r racingRecent) Recent(ctx context.Context) ([]Call, error) {
// **A finish recorded before the sweep reaches a call stands, with its kept answer** (novox/hq issue
// 711): a stale "running" snapshot is no excuse to overwrite a record that has since said how the
// call ended. A caller reading `abandoned` for a call that succeeded would redo work already done.
func TestNatsAFinishRacingTheSweepsSnapshotReadsFinished(t *testing.T) {
keeper := callsBucket(t)
running := Call{ID: "call-711-1", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"}
if err := keeper.Keep(t.Context(), running); err != nil {
t.Fatal(err)
}
waitKept(t, keeper, running.ID, CallRunning)
finished := running
finished.State = CallAnswered
finished.Answer = json.RawMessage(`"anchor told"`)
wrapped := racingRecent{CallKeeper: keeper, thenLand: func() {
if err := keeper.Keep(context.Background(), finished); err != nil {
t.Error(err)
}
waitKept(t, keeper, running.ID, CallAnswered)
}}
var said bytes.Buffer
second := NewCallLog()
if err := second.Durably(t.Context(), wrapped, "controller@two", log.New(&said, "", 0)); err != nil {
t.Fatal(err)
}
got, found, err := second.Get(running.ID)
if err != nil || !found || got.State != CallAnswered || !strings.Contains(string(got.Answer), "anchor told") {
t.Fatalf("its finish did not stand: %+v %v %v", got, found, err)
}
if strings.Contains(said.String(), running.ID) {
t.Errorf("a finished call was said abandoned: %q", said.String())
}
}
// A call left running with no finish recorded is still marked abandoned — unchanged (novox/hq issue
// 711): only a finish actually recorded for it may stand instead.
func TestNatsARunningCallWithNoFinishIsStillAbandoned(t *testing.T) {
keeper := callsBucket(t)
running := Call{ID: "call-711-2", Seat: "mesh-controller", Verb: "plans", State: CallRunning, Holder: "controller@one"}
if err := keeper.Keep(t.Context(), running); err != nil {
t.Fatal(err)
}
waitKept(t, keeper, running.ID, CallRunning)
var said bytes.Buffer
second := NewCallLog()
if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil {
t.Fatal(err)
}
got, found, err := second.Get(running.ID)
if err != nil || !found || got.State != CallAbandoned || !strings.Contains(said.String(), running.ID) {
t.Fatalf("got %+v %v %v; said %q", got, found, err, said.String())
}
}
// delayedKV wraps a jetstream.KeyValue so a hook runs just before Update, to land a write between a
// reader's Get and its own compare-and-set write.
@@ -193,42 +70,6 @@ func (d delayedKV) Update(ctx context.Context, key string, value []byte, revisio
// **The race where a finish lands after the sweep's own read but before its write: the finish
// wins** (novox/hq issue 711). The sweep's write is a compare-and-set on the revision it read, so a
// finish landing in between invalidates it; the sweep rereads, finds the finish, and leaves it alone.
func TestNatsAFinishRacingTheSweepsWriteWins(t *testing.T) {
keeper := callsBucket(t)
running := Call{ID: "call-711-3", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"}
if err := keeper.Keep(t.Context(), running); err != nil {
t.Fatal(err)
}
waitKept(t, keeper, running.ID, CallRunning)
finished := running
finished.State = CallFinishedAfter
finished.Answer = json.RawMessage(`"anchor told"`)
landed := false
racey := &BusCalls{kv: delayedKV{KeyValue: keeper.kv, before: func() {
if landed {
return
}
landed = true
if err := keeper.Keep(context.Background(), finished); err != nil {
t.Error(err)
}
waitKept(t, keeper, running.ID, CallFinishedAfter)
}}}
var said bytes.Buffer
second := NewCallLog()
if err := second.Durably(t.Context(), racey, "controller@two", log.New(&said, "", 0)); err != nil {
t.Fatal(err)
}
got, found, err := second.Get(running.ID)
if err != nil || !found || got.State != CallFinishedAfter || !strings.Contains(string(got.Answer), "anchor told") {
t.Fatalf("the finish did not win the race: %+v %v %v", got, found, err)
}
if strings.Contains(said.String(), running.ID) {
t.Errorf("a finished call was said abandoned: %q", said.String())
}
}
// alwaysMismatchKV wraps a jetstream.KeyValue whose Update always fails with a revision mismatch, as
// if some other writer kept winning the race forever (novox/hq issue 711 review round 1).
@@ -245,55 +86,11 @@ func (a alwaysMismatchKV) Update(ctx context.Context, key string, value []byte,
// **MarkAbandoned gives up after markAbandonedTries rather than loop forever** (novox/hq issue 711
// review round 1): a record whose compare-and-set keeps losing is left as it stands, said in an
// error naming the call, not retried without end.
func TestNatsAMarkAbandonedGivesUpAfterTenTries(t *testing.T) {
keeper := callsBucket(t)
running := Call{ID: "call-711-4", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"}
if err := keeper.Keep(t.Context(), running); err != nil {
t.Fatal(err)
}
waitKept(t, keeper, running.ID, CallRunning)
tries := 0
stuck := &BusCalls{kv: alwaysMismatchKV{KeyValue: keeper.kv, tries: &tries}}
_, err := stuck.MarkAbandoned(t.Context(), running.ID)
if err == nil || !strings.Contains(err.Error(), running.ID) {
t.Fatalf("expected an error naming %s, got %v", running.ID, err)
}
if tries != markAbandonedTries {
t.Fatalf("tried %d times, want %d", tries, markAbandonedTries)
}
}
// The bucket holds the last thousand calls or fourteen days: a call is two keys, so the stream under
// it holds twice as many messages, and asserting it again keeps the count.
func TestNatsTheCallsBucketIsBounded(t *testing.T) {
callsBucket(t)
js := aBus(t)
if err := js.EnsureControllerBuckets(); err != nil {
t.Fatal(err)
}
info, err := js.Context().StreamInfo("KV_" + broker.CallsBucket)
if err != nil {
t.Fatal(err)
}
if info.Config.MaxMsgs != 2*broker.KeptCallsDurably || info.Config.MaxAge != broker.CallsKeptFor {
t.Fatalf("kept %d messages for %s", info.Config.MaxMsgs, info.Config.MaxAge)
}
}
// An answer larger than the bound is cut and says so, and the record is still written.
func TestNatsAnAnswerLargerThanTheBoundIsCut(t *testing.T) {
keeper := callsBucket(t)
big, _ := json.Marshal(map[string]string{"result": strings.Repeat("x", broker.CallAnswerBytes+10)})
c := Call{ID: "call-1-1", Seat: "mesh-controller", Verb: "plan", State: CallAnswered, Answer: big}
if err := keeper.Keep(t.Context(), c); err != nil {
t.Fatal(err)
}
got, found, err := keeper.Kept(t.Context(), "call-1-1")
if err != nil || !found || !strings.Contains(string(got.Answer), "the first") {
t.Fatalf("%v %v %.200s", found, err, got.Answer)
}
}
// Who asked is read from the inbox the answer goes to.
func TestTheCallerIsTheInboxsPrincipal(t *testing.T) {
internal/link/calls_test.go
-27
View File
@@ -12,10 +12,6 @@ import (
"testing"
"time"
"unicode"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// answers collects what a call answered, and fails a second answer: the bus permits one.
@@ -280,26 +276,3 @@ func TestAJoinTokenIsShownToItsCallerAndNotKept(t *testing.T) {
// A verb whose handler panics answers an error, and the controller serves the next call (review of
// novox/hq issue 327: read verbs are answered in the serving process now).
func TestAHandlerThatPanicsAnswersAnError(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
stop, err := OverNATS{Conn: conn}.ServeSeatTools("panicky", map[string]ToolHandler{
"boom": func(context.Context, json.RawMessage) (any, error) { panic("nil map") },
"fine": func(context.Context, json.RawMessage) (any, error) { return "ok", nil },
}, nil)
if err != nil {
t.Fatal(err)
}
defer stop()
answer, err := AskMeshSeatTool(context.Background(), conn, "panicky", "boom", map[string]any{}, 5*time.Second)
if err != nil || !strings.Contains(answer.Error, "failed inside the controller") {
t.Fatalf("a panic answered %+v (%v)", answer, err)
}
answer, err = AskMeshSeatTool(context.Background(), conn, "panicky", "fine", map[string]any{}, 5*time.Second)
if err != nil || answer.Error != "" {
t.Fatalf("the call after a panic answered %+v (%v)", answer, err)
}
}
internal/link/check_builds_integration_test.go
+67
View File
@@ -0,0 +1,67 @@
//go:build integration
package link
import (
"encoding/json"
"testing"
"time"
)
func TestNatsTheBuildsAWalkMayTakeAreTheChecksOutcomesOfItsPullRequests(t *testing.T) {
js := aBus(t)
since := time.Now().Add(-time.Minute)
say := func(r BuildResult) {
body, _ := json.Marshal(r)
if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil {
t.Fatal(err)
}
}
mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7}
built := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}}
say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built,
{Module: "web", NotBuilt: "it has no reuse key"}}}})
say(BuildResult{ID: "build-2", Checked: &CheckRequest{Owner: "novox", Repo: "shop", Number: 8},
Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built}}})
say(BuildResult{ID: "build-3", Repository: "novox/shop", Commit: "c0ffee"}) // a build, not a check
newer := built
newer.Key = "r1:2"
say(BuildResult{ID: "build-4", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{newer}}})
got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since)
if err != nil {
t.Fatal(err)
}
if len(got) != 2 || got[0].Check != "build-4" || got[0].Built.Key != "r1:2" || got[1].Check != "build-1" ||
got[1].Pull.Number != 7 {
t.Fatalf("read %+v", got)
}
}
func TestNatsAKeptBuildIsTheBuildersCheckAndEveryBuildSaysItsHead(t *testing.T) {
js := aBus(t)
since := time.Now().Add(-time.Minute)
say := func(r BuildResult) {
body, _ := json.Marshal(r)
if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil {
t.Fatal(err)
}
}
mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7}
app := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}}
say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa",
Built: []CheckBuilt{app}}})
kept := app
kept.Kept = "build-1"
say(BuildResult{ID: "build-2", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa",
Built: []CheckBuilt{kept}}})
got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since)
if err != nil {
t.Fatal(err)
}
if len(got) != 2 || got[0].Check != "build-1" || got[0].Built.Kept != "build-1" || got[0].Head != "aaaaaaaaaaaa" ||
got[1].Check != "build-1" || got[1].Head != "aaaaaaaaaaaa" {
t.Fatalf("read %+v", got)
}
}
internal/link/check_builds_test.go
-58
View File
@@ -1,72 +1,14 @@
package link
import (
"encoding/json"
"testing"
"time"
)
// The builds a walk may take are read from the build seat's own outcomes of its checks (novox/hq ADR 0350 §2): only
// checks of the walk's pull requests, only builds made, newest first.
func TestNatsTheBuildsAWalkMayTakeAreTheChecksOutcomesOfItsPullRequests(t *testing.T) {
js := aBus(t)
since := time.Now().Add(-time.Minute)
say := func(r BuildResult) {
body, _ := json.Marshal(r)
if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil {
t.Fatal(err)
}
}
mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7}
built := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}}
say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built,
{Module: "web", NotBuilt: "it has no reuse key"}}}})
say(BuildResult{ID: "build-2", Checked: &CheckRequest{Owner: "novox", Repo: "shop", Number: 8},
Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built}}})
say(BuildResult{ID: "build-3", Repository: "novox/shop", Commit: "c0ffee"}) // a build, not a check
newer := built
newer.Key = "r1:2"
say(BuildResult{ID: "build-4", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{newer}}})
got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since)
if err != nil {
t.Fatal(err)
}
if len(got) != 2 || got[0].Check != "build-4" || got[0].Built.Key != "r1:2" || got[1].Check != "build-1" ||
got[1].Pull.Number != 7 {
t.Fatalf("read %+v", got)
}
}
// A build is read with the head its check judged, and a kept build is named as the check's that built it (novox/hq
// issue 703).
func TestNatsAKeptBuildIsTheBuildersCheckAndEveryBuildSaysItsHead(t *testing.T) {
js := aBus(t)
since := time.Now().Add(-time.Minute)
say := func(r BuildResult) {
body, _ := json.Marshal(r)
if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil {
t.Fatal(err)
}
}
mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7}
app := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}}
say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa",
Built: []CheckBuilt{app}}})
kept := app
kept.Kept = "build-1"
say(BuildResult{ID: "build-2", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa",
Built: []CheckBuilt{kept}}})
got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since)
if err != nil {
t.Fatal(err)
}
if len(got) != 2 || got[0].Check != "build-1" || got[0].Built.Kept != "build-1" || got[0].Head != "aaaaaaaaaaaa" ||
got[1].Check != "build-1" || got[1].Head != "aaaaaaaaaaaa" {
t.Fatalf("read %+v", got)
}
}
// Two spellings of a head are the same head only when both say at least twelve characters of it.
func TestSameHeadNeedsTwelveCharactersOfBoth(t *testing.T) {
internal/link/checks_nats_integration_test.go
+512
View File
@@ -0,0 +1,512 @@
//go:build integration
package link
import (
"context"
"encoding/json"
"reflect"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false),
aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false),
aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk
aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false),
aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
var got []string
for _, a := range queueOf(t, js).Asks {
if a.State != AskWaiting {
t.Errorf("%s is %s", a.ID, a.State)
}
got = append(got, a.ID)
}
if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) {
t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want)
}
}
func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("other-1", 1, "h1", "t", false),
aPullCheck("other-2", 2, "h2", "t", false),
aPullCheck("ready-3", 3, "h3", "t", true),
{ID: "delivery-4", Repository: "/d"},
aPullCheck("other-5", 5, "h5", "t", false),
aPullCheck("ready-6", 6, "h6", "t", true),
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
took := make(chan string, 8)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
var order []string
for len(order) < 6 {
select {
case id := <-took:
order = append(order, id)
case <-time.After(15 * time.Second):
t.Fatalf("taken so far: %v", order)
}
}
if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) {
t.Fatalf("taken as %v; want %v", order, want)
}
}
func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aKeyBucket(t, js)
ctx, stop := context.WithCancel(t.Context())
defer stop()
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged
putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false),
aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false),
aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false),
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
took := make(chan string, 8)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
var ran []string
for len(ran) < 2 {
select {
case id := <-took:
ran = append(ran, id)
case <-time.After(15 * time.Second):
t.Fatalf("run so far: %v", ran)
}
}
if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) {
t.Fatalf("run %v; want %v, the judged one dropped", ran, want)
}
time.Sleep(300 * time.Millisecond)
if n := len(queueOf(t, js).Asks); n != 0 {
t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n)
}
}
func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aKeyBucket(t, js)
ctx, stop := context.WithCancel(t.Context())
defer stop()
moved := make(chan PullKey, 4)
req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false)
end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k })
if err != nil {
t.Fatal(err)
}
defer end()
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"})
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
select {
case k := <-moved:
t.Fatalf("stopped by a key that moved nothing: %+v", k)
case <-time.After(500 * time.Millisecond):
}
putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"})
select {
case k := <-moved:
if k.Head != "head-7-111111111" {
t.Errorf("stopped by %+v", k)
}
case <-time.After(5 * time.Second):
t.Fatal("a key whose head moved did not stop the check")
}
// And the trunk commit.
req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false)
end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k })
if err != nil {
t.Fatal(err)
}
defer end2()
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"})
select {
case <-moved:
case <-time.After(5 * time.Second):
t.Fatal("a key whose trunk commit moved did not stop the check")
}
// A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does
// have is the pull request's own.
group := BuildRequest{ID: "group-1", Ref: "head-7-000000000",
Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}}
end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k })
if err != nil {
t.Fatal(err)
}
defer end3()
putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"})
select {
case k := <-moved:
t.Fatalf("a group's check was stopped by %+v", k)
case <-time.After(500 * time.Millisecond):
}
}
func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("other-1", 1, "h1", "t", false),
aPullCheck("ready-2", 2, "h2", "t", true),
{ID: "delivery-3", Repository: "/d"},
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
q := queueOf(t, js)
var ids, streams []string
for _, a := range q.Asks {
ids, streams = append(ids, a.ID), append(streams, a.Stream)
}
if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) {
t.Fatalf("read %v; want the order they are handed out, %v", ids, want)
}
if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) {
t.Fatalf("streams %v; want %v", streams, want)
}
ready, _ := q.Find("ready-2")
if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker {
t.Errorf("the ready check is handed out by %s", w.Name)
}
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq {
t.Fatalf("deleted %d, %v", removed, err)
}
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 {
t.Fatalf("deleted again: %d, %v", removed, err)
}
if n := len(queueOf(t, js).Asks); n != 2 {
t.Errorf("%d asks left; want 2", n)
}
}
func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) {
js := aBusWithTheBuildRole(t)
aKeyBucket(t, js)
aRunningBucket(t, js, 2*time.Minute)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
running := make(chan string, 4)
release := make(chan struct{})
a := MachineOverNATS(js, "builder-a")
defer a.Close()
go func() {
_ = a.Take(ctx, func(ctx context.Context, work Build) {
running <- work.Request().ID
<-release
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
_ = work.Done()
})
}()
if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
select {
case <-running:
case <-time.After(10 * time.Second):
t.Fatal("builder a took nothing")
}
// The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it.
if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
b := MachineOverNATS(js, "builder-b")
defer b.Close()
took := make(chan string, 4)
go func() {
_ = b.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
_ = work.Done()
})
}()
select {
case id := <-took:
t.Fatalf("builder b took %s, which builder a is running a check of", id)
case <-time.After(3 * time.Second):
}
if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n)
}
// Another trunk commit is another check.
if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
if id != "moved" {
t.Fatalf("builder b took %s", id)
}
case <-time.After(10 * time.Second):
t.Fatal("a check against another trunk commit was dropped too")
}
close(release)
}
func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) {
js := aBusWithTheBuildRole(t)
aKeyBucket(t, js)
kv := aRunningBucket(t, js, 2*time.Second)
ctx, stop := context.WithCancel(t.Context())
defer stop()
body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"})
if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil {
t.Fatal(err)
}
asks := BuildsOn(js, TheBuildMachine)
took := make(chan string, 4)
b := MachineOverNATS(js, "builder-b")
defer b.Close()
go func() {
_ = b.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
_ = work.Done()
})
}()
if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
t.Fatalf("%s was taken while the key was fresh", id)
case <-time.After(1200 * time.Millisecond):
}
time.Sleep(2500 * time.Millisecond) // the key was not renewed
if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
if id != "after-it-aged" {
t.Fatalf("took %s", id)
}
case <-time.After(10 * time.Second):
t.Fatal("the request was not taken after the key aged out")
}
}
func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aRunningBucket(t, js, 2*time.Second)
was := heartbeatWhileWorking
heartbeatWhileWorking = 400 * time.Millisecond
t.Cleanup(func() { heartbeatWhileWorking = was })
ctx, stop := context.WithCancel(t.Context())
defer stop()
running := make(chan struct{}, 1)
release := make(chan struct{})
m := MachineOverNATS(js, "builder-a")
defer m.Close()
go func() {
_ = m.Take(ctx, func(ctx context.Context, work Build) {
running <- struct{}{}
<-release
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
_ = work.Done()
})
}()
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
<-running
key := PullKeyOf("novox", "a", 7)
e, err := kv.Get(ctx, key)
if err != nil {
t.Fatalf("no key while the check runs: %v", err)
}
var got CheckRunning
_ = json.Unmarshal(e.Value(), &got)
if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" {
t.Errorf("the key says %+v", got)
}
time.Sleep(3 * time.Second) // longer than the bucket's age
if _, err := kv.Get(ctx, key); err != nil {
t.Fatalf("the key aged out under a check still running: %v", err)
}
close(release)
for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) {
if _, err := kv.Get(ctx, key); err != nil {
break
}
if time.Now().After(end) {
t.Fatal("the key stays after the check ended")
}
}
}
func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aKeyBucket(t, js)
aRunningBucket(t, js, 2*time.Minute)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
outcomes := make(chan BuildResult, 8)
watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) {
var r BuildResult
if json.Unmarshal(m.Data, &r) == nil {
outcomes <- r
}
})
if err != nil {
t.Fatal(err)
}
defer func() { _ = watching.Unsubscribe() }()
ran := make(chan string, 4)
m := MachineOverNATS(js, "builder-a")
defer m.Close()
go func() {
_ = m.Take(ctx, func(ctx context.Context, work Build) {
building, cancel := context.WithCancel(ctx)
defer cancel()
ran <- work.Request().ID
stopped := make(chan struct{}, 1)
end, err := work.(interface {
WatchPull(context.Context, func(PullKey)) (func(), error)
}).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() })
if err != nil {
t.Error(err)
return
}
defer end()
if work.Request().ID == "older" {
select {
case <-stopped:
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey})
case <-time.After(15 * time.Second):
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"})
}
} else {
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}})
}
_ = work.Done()
})
}()
if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
if id := <-ran; id != "older" {
t.Fatalf("took %s", id)
}
time.Sleep(300 * time.Millisecond) // the watch is made
// The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked.
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"})
if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-ran:
if id != "newer" {
t.Fatalf("took %s", id)
}
case <-time.After(15 * time.Second):
t.Fatal("the newer request, for the same head on the new trunk commit, was not taken")
}
got := map[string]string{}
for len(got) < 2 {
select {
case r := <-outcomes:
got[r.ID] = r.Failed
case <-time.After(10 * time.Second):
t.Fatalf("outcomes so far: %v", got)
}
}
if got["older"] != SupersededByKey || got["newer"] != "" {
t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got)
}
}
func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
for _, ready := range []bool{false, true} {
js := aBusWithTheBuildRole(t)
name, worker := "other", broker.ChecksOtherWorker
if ready {
name, worker = "ready", broker.ChecksReadyWorker
}
// A worker whose wait is a second, as the mesh's is a minute.
for _, w := range broker.CheckWorkers() {
if w.Name == worker {
w.AckWaitSeconds = 1
if err := js.EnsureConsumer(w); err != nil {
t.Fatal(err)
}
}
}
ctx, stop := context.WithCancel(t.Context())
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil {
t.Fatal(err)
}
// The first builder takes it and dies: its pull is fetched, never acknowledged.
if _, err := fetchOne(js, worker); err != nil {
t.Fatal(err)
}
took := make(chan BuildRequest, 2)
second := MachineOverNATS(js, "builder-b")
go func() {
_ = second.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request()
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
_ = work.Done()
})
}()
select {
case r := <-took:
if r.ID != "dies-"+name {
t.Errorf("%s: took %s", name, r.ID)
}
case <-time.After(15 * time.Second):
t.Errorf("%s: the unacknowledged check did not go to a second builder", name)
}
second.Close()
stop()
}
}
internal/link/checks_nats_test.go
-489
View File
@@ -3,7 +3,6 @@ package link
import (
"context"
"encoding/json"
"reflect"
"testing"
"time"
@@ -56,75 +55,9 @@ func queueOf(t *testing.T, js *broker.JetStream) Queue {
// A newer request for a pull request replaces the one still waiting, by the stream's rule, and each open pull request is
// left with one queued check (acceptance 5).
func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false),
aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false),
aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk
aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false),
aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
var got []string
for _, a := range queueOf(t, js).Asks {
if a.State != AskWaiting {
t.Errorf("%s is %s", a.ID, a.State)
}
got = append(got, a.ID)
}
if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) {
t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want)
}
}
// A delivery build is taken before a check of a ready pull request, that before any other check, and the others in the
// order they were asked.
func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("other-1", 1, "h1", "t", false),
aPullCheck("other-2", 2, "h2", "t", false),
aPullCheck("ready-3", 3, "h3", "t", true),
{ID: "delivery-4", Repository: "/d"},
aPullCheck("other-5", 5, "h5", "t", false),
aPullCheck("ready-6", 6, "h6", "t", true),
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
took := make(chan string, 8)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
var order []string
for len(order) < 6 {
select {
case id := <-took:
order = append(order, id)
case <-time.After(15 * time.Second):
t.Fatalf("taken so far: %v", order)
}
}
if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) {
t.Fatalf("taken as %v; want %v", order, want)
}
}
// aKeyBucket makes mesh-delivery's bucket of pull requests' keys, which the bus here has not been given by that module.
func aKeyBucket(t *testing.T, js *broker.JetStream) jetstream.KeyValue {
@@ -150,113 +83,9 @@ func putKey(t *testing.T, kv jetstream.KeyValue, number int, k PullKey) {
// A request left on the third queue for a head and a trunk commit its key says were judged is acknowledged and dropped when a
// builder takes it; one for a head the key does not say judged is run.
func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aKeyBucket(t, js)
ctx, stop := context.WithCancel(t.Context())
defer stop()
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged
putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false),
aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false),
aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false),
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
took := make(chan string, 8)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
var ran []string
for len(ran) < 2 {
select {
case id := <-took:
ran = append(ran, id)
case <-time.After(15 * time.Second):
t.Fatalf("run so far: %v", ran)
}
}
if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) {
t.Fatalf("run %v; want %v, the judged one dropped", ran, want)
}
time.Sleep(300 * time.Millisecond)
if n := len(queueOf(t, js).Asks); n != 0 {
t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n)
}
}
// A running check stops when its key's head moves or its trunk commit does, not when the key only gains its verdict, and a
// group's check, which a member's merge in the group's order writes nothing to the key of, is not stopped at all.
func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aKeyBucket(t, js)
ctx, stop := context.WithCancel(t.Context())
defer stop()
moved := make(chan PullKey, 4)
req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false)
end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k })
if err != nil {
t.Fatal(err)
}
defer end()
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"})
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"})
select {
case k := <-moved:
t.Fatalf("stopped by a key that moved nothing: %+v", k)
case <-time.After(500 * time.Millisecond):
}
putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"})
select {
case k := <-moved:
if k.Head != "head-7-111111111" {
t.Errorf("stopped by %+v", k)
}
case <-time.After(5 * time.Second):
t.Fatal("a key whose head moved did not stop the check")
}
// And the trunk commit.
req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false)
end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k })
if err != nil {
t.Fatal(err)
}
defer end2()
putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"})
select {
case <-moved:
case <-time.After(5 * time.Second):
t.Fatal("a key whose trunk commit moved did not stop the check")
}
// A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does
// have is the pull request's own.
group := BuildRequest{ID: "group-1", Ref: "head-7-000000000",
Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}}
end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k })
if err != nil {
t.Fatal(err)
}
defer end3()
putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"})
select {
case k := <-moved:
t.Fatalf("a group's check was stopped by %+v", k)
case <-time.After(500 * time.Millisecond):
}
}
func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) {
k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: "pass"}
@@ -270,44 +99,6 @@ func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) {
// The queue is read across the three streams in the order they are handed out, each ask says the stream it waits in, and an
// ask is removed from that stream.
func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) {
js := aBusWithTheBuildRole(t)
ctx := t.Context()
asks := BuildsOn(js, TheBuildMachine)
for _, r := range []BuildRequest{
aPullCheck("other-1", 1, "h1", "t", false),
aPullCheck("ready-2", 2, "h2", "t", true),
{ID: "delivery-3", Repository: "/d"},
} {
if err := asks.Ask(ctx, r); err != nil {
t.Fatal(err)
}
}
q := queueOf(t, js)
var ids, streams []string
for _, a := range q.Asks {
ids, streams = append(ids, a.ID), append(streams, a.Stream)
}
if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) {
t.Fatalf("read %v; want the order they are handed out, %v", ids, want)
}
if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) {
t.Fatalf("streams %v; want %v", streams, want)
}
ready, _ := q.Find("ready-2")
if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker {
t.Errorf("the ready check is handed out by %s", w.Name)
}
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq {
t.Fatalf("deleted %d, %v", removed, err)
}
if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 {
t.Fatalf("deleted again: %d, %v", removed, err)
}
if n := len(queueOf(t, js).Asks); n != 2 {
t.Errorf("%d asks left; want 2", n)
}
}
// aRunningBucket makes the bucket the builders say the checks they run in, with an age of its own short enough for a test
// (the mesh's is two minutes): a key not renewed within it is gone.
@@ -326,165 +117,12 @@ func aRunningBucket(t *testing.T, js *broker.JetStream, age time.Duration) jetst
// A builder taking a request for a pull request's head and trunk commit that another builder is running a check of drops it:
// the bucket of running checks names the head and the trunk, the builder and the build (issue 974).
func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) {
js := aBusWithTheBuildRole(t)
aKeyBucket(t, js)
aRunningBucket(t, js, 2*time.Minute)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
running := make(chan string, 4)
release := make(chan struct{})
a := MachineOverNATS(js, "builder-a")
defer a.Close()
go func() {
_ = a.Take(ctx, func(ctx context.Context, work Build) {
running <- work.Request().ID
<-release
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
_ = work.Done()
})
}()
if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
select {
case <-running:
case <-time.After(10 * time.Second):
t.Fatal("builder a took nothing")
}
// The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it.
if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
b := MachineOverNATS(js, "builder-b")
defer b.Close()
took := make(chan string, 4)
go func() {
_ = b.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
_ = work.Done()
})
}()
select {
case id := <-took:
t.Fatalf("builder b took %s, which builder a is running a check of", id)
case <-time.After(3 * time.Second):
}
if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 {
t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n)
}
// Another trunk commit is another check.
if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
if id != "moved" {
t.Fatalf("builder b took %s", id)
}
case <-time.After(10 * time.Second):
t.Fatal("a check against another trunk commit was dropped too")
}
close(release)
}
// A key that is not renewed ages out, and the request for its head and trunk commit is then taken: a builder that died
// running a check leaves nothing that holds the next one back for longer than the bucket's age.
func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) {
js := aBusWithTheBuildRole(t)
aKeyBucket(t, js)
kv := aRunningBucket(t, js, 2*time.Second)
ctx, stop := context.WithCancel(t.Context())
defer stop()
body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"})
if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil {
t.Fatal(err)
}
asks := BuildsOn(js, TheBuildMachine)
took := make(chan string, 4)
b := MachineOverNATS(js, "builder-b")
defer b.Close()
go func() {
_ = b.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
_ = work.Done()
})
}()
if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
t.Fatalf("%s was taken while the key was fresh", id)
case <-time.After(1200 * time.Millisecond):
}
time.Sleep(2500 * time.Millisecond) // the key was not renewed
if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
if id != "after-it-aged" {
t.Fatalf("took %s", id)
}
case <-time.After(10 * time.Second):
t.Fatal("the request was not taken after the key aged out")
}
}
// A builder running a check renews its key on each heartbeat, so it outlives the bucket's age, and deletes it when the check
// ends, whatever the outcome.
func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aRunningBucket(t, js, 2*time.Second)
was := heartbeatWhileWorking
heartbeatWhileWorking = 400 * time.Millisecond
t.Cleanup(func() { heartbeatWhileWorking = was })
ctx, stop := context.WithCancel(t.Context())
defer stop()
running := make(chan struct{}, 1)
release := make(chan struct{})
m := MachineOverNATS(js, "builder-a")
defer m.Close()
go func() {
_ = m.Take(ctx, func(ctx context.Context, work Build) {
running <- struct{}{}
<-release
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"})
_ = work.Done()
})
}()
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
<-running
key := PullKeyOf("novox", "a", 7)
e, err := kv.Get(ctx, key)
if err != nil {
t.Fatalf("no key while the check runs: %v", err)
}
var got CheckRunning
_ = json.Unmarshal(e.Value(), &got)
if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" {
t.Errorf("the key says %+v", got)
}
time.Sleep(3 * time.Second) // longer than the bucket's age
if _, err := kv.Get(ctx, key); err != nil {
t.Fatalf("the key aged out under a check still running: %v", err)
}
close(release)
for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) {
if _, err := kv.Get(ctx, key); err != nil {
break
}
if time.Now().After(end) {
t.Fatal("the key stays after the check ended")
}
}
}
// A key whose verdict is "error" — a check that could not run, or was stopped — judged nothing: a request for that head and
// trunk commit must run (issue 974, review 2).
@@ -499,136 +137,9 @@ func TestAnErrorVerdictJudgesNothing(t *testing.T) {
// A check stopped because the trunk moved leaves the newer request, for the head against the new trunk commit, taken and
// run: the stop is the stopped one's failure, not the key's verdict, and the newer request is not dropped by it.
func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) {
js := aBusWithTheBuildRole(t)
kv := aKeyBucket(t, js)
aRunningBucket(t, js, 2*time.Minute)
ctx, stop := context.WithCancel(t.Context())
defer stop()
asks := BuildsOn(js, TheBuildMachine)
outcomes := make(chan BuildResult, 8)
watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) {
var r BuildResult
if json.Unmarshal(m.Data, &r) == nil {
outcomes <- r
}
})
if err != nil {
t.Fatal(err)
}
defer func() { _ = watching.Unsubscribe() }()
ran := make(chan string, 4)
m := MachineOverNATS(js, "builder-a")
defer m.Close()
go func() {
_ = m.Take(ctx, func(ctx context.Context, work Build) {
building, cancel := context.WithCancel(ctx)
defer cancel()
ran <- work.Request().ID
stopped := make(chan struct{}, 1)
end, err := work.(interface {
WatchPull(context.Context, func(PullKey)) (func(), error)
}).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() })
if err != nil {
t.Error(err)
return
}
defer end()
if work.Request().ID == "older" {
select {
case <-stopped:
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey})
case <-time.After(15 * time.Second):
_ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"})
}
} else {
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}})
}
_ = work.Done()
})
}()
if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil {
t.Fatal(err)
}
if id := <-ran; id != "older" {
t.Fatalf("took %s", id)
}
time.Sleep(300 * time.Millisecond) // the watch is made
// The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked.
putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"})
if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil {
t.Fatal(err)
}
select {
case id := <-ran:
if id != "newer" {
t.Fatalf("took %s", id)
}
case <-time.After(15 * time.Second):
t.Fatal("the newer request, for the same head on the new trunk commit, was not taken")
}
got := map[string]string{}
for len(got) < 2 {
select {
case r := <-outcomes:
got[r.ID] = r.Failed
case <-time.After(10 * time.Second):
t.Fatalf("outcomes so far: %v", got)
}
}
if got["older"] != SupersededByKey || got["newer"] != "" {
t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got)
}
}
// An unacknowledged check goes to a second builder: taken from a check worker and not acknowledged, it is handed out again
// when the worker's wait is up.
func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) {
for _, ready := range []bool{false, true} {
js := aBusWithTheBuildRole(t)
name, worker := "other", broker.ChecksOtherWorker
if ready {
name, worker = "ready", broker.ChecksReadyWorker
}
// A worker whose wait is a second, as the mesh's is a minute.
for _, w := range broker.CheckWorkers() {
if w.Name == worker {
w.AckWaitSeconds = 1
if err := js.EnsureConsumer(w); err != nil {
t.Fatal(err)
}
}
}
ctx, stop := context.WithCancel(t.Context())
if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil {
t.Fatal(err)
}
// The first builder takes it and dies: its pull is fetched, never acknowledged.
if _, err := fetchOne(js, worker); err != nil {
t.Fatal(err)
}
took := make(chan BuildRequest, 2)
second := MachineOverNATS(js, "builder-b")
go func() {
_ = second.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request()
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"})
_ = work.Done()
})
}()
select {
case r := <-took:
if r.ID != "dies-"+name {
t.Errorf("%s: took %s", name, r.ID)
}
case <-time.After(15 * time.Second):
t.Errorf("%s: the unacknowledged check did not go to a second builder", name)
}
second.Close()
stop()
}
}
// fetchOne takes the next message of a worker and leaves it unacknowledged, as a builder that died would.
func fetchOne(js *broker.JetStream, worker string) (*nats.Msg, error) {
internal/link/conformance_integration_test.go
+148
View File
@@ -0,0 +1,148 @@
//go:build integration
package link
import (
"bytes"
"encoding/json"
"fmt"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/beside"
)
func loadFixture(t *testing.T, name string) fixture {
t.Helper()
raw, from, err := sdkFixture(os.Getenv(beside.Env), goModFile, capturedSDK, capturedLog, name)
if err != nil {
// Failed, never skipped: a skip here passed the suite with nothing judged.
t.Fatal(err)
}
t.Logf("%s: %s", name, from)
if strings.HasPrefix(from, notJudged) {
// Said on the check's own output too, where `go test` without -v prints no log of a passing test.
saidNotJudged.Do(func() { fmt.Fprintln(os.Stderr, "internal/link conformance: "+from) })
}
var f fixture
if err := json.Unmarshal(raw, &f); err != nil {
t.Fatalf("%s: %v", name, err)
}
return f
}
func sdkFixture(root, goMod, captured, capturedLog, name string) ([]byte, string, error) {
file := path.Join("conformance", name)
readCaptured := func() ([]byte, error) {
raw, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file)))
if err != nil {
return nil, fmt.Errorf("the captured SDK's %s: %w", file, err)
}
return raw, nil
}
if root == "" {
raw, err := readCaptured()
return raw, "the copy captured in testdata/beside (see CAPTURED); a merge check reads the SDK's clone " +
"at this tree's pin", err
}
clone := filepath.Join(root, "mesh-sdk")
if _, err := os.Stat(clone); err != nil {
// A build seat asked by a controller from before mesh-controller #220 places no mesh-sdk beside the
// check, so #220 could never pass its own check if this failed. Until #220 has rolled out, the
// captured copy is read and the log says so loudly; the follow-up of novox/hq issue 449 makes this
// a failure again.
at, cerr := capturedCommit(capturedLog)
if cerr != nil {
return nil, "", cerr
}
raw, rerr := readCaptured()
return raw, fmt.Sprintf(notJudged+": the seat placed no mesh-sdk beside this check in "+
"%s=%s (it does once mesh-controller #220 has rolled out); read the captured copy at %s", beside.Env,
root, at), rerr
}
ref, err := sdkRef(goMod)
if err != nil {
return nil, "", err
}
raw, err := gitShow(clone, ref, file)
if err != nil {
return nil, "", fmt.Errorf("the mesh-sdk clone beside this check has no %s at %s, the SDK this tree's "+
"go.mod pins: %w", file, ref, err)
}
// The captured copy is the SDK's own, never edited by hand: the clone at the captured commit says so.
at, err := capturedCommit(capturedLog)
if err != nil {
return nil, "", err
}
theirs, err := gitShow(clone, at, file)
if err != nil {
return nil, "", fmt.Errorf("the mesh-sdk clone has no %s at %s, the commit CAPTURED names: %w", file, at, err)
}
ours, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file)))
if err != nil {
return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err)
}
if !bytes.Equal(ours, theirs) {
return nil, "", fmt.Errorf("the captured %s is not mesh-sdk's at %s, the commit CAPTURED names: it was "+
"edited or captured wrong; capture it again as CAPTURED says", file, at)
}
return raw, "mesh-sdk cloned beside this check, at " + ref + " (this tree's go.mod)", nil
}
func gitShow(repository, ref, file string) ([]byte, error) {
cmd := exec.Command("git", "-c", "safe.directory=*", "-C", repository, "show", ref+":"+file)
var stderr bytes.Buffer
cmd.Stderr = &stderr
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("git show %s:%s: %v: %s", ref, file, err, strings.TrimSpace(stderr.String()))
}
return out, nil
}
func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) {
f := loadFixture(t, "events/module-event.json")
sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node)
for _, want := range f.Wire.RequiredHeaders {
if _, ok := sent[want]; !ok {
t.Errorf("the Go emitter does not set %q, which the fixture requires — an event it "+
"emits is one a conforming consumer refuses", want)
}
}
}
func TestTheGoEmittersHeaderFormatsMatch(t *testing.T) {
f := loadFixture(t, "events/module-event.json")
sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node)
if got := sent["content-type"]; got != f.Wire.HeaderFormats["content-type"] {
t.Errorf("content-type is %q, the fixture says %q", got, f.Wire.HeaderFormats["content-type"])
}
if _, err := time.Parse(time.RFC3339, sent["x-time"]); err != nil {
t.Errorf("x-time %q is not RFC3339, which the fixture requires: %v", sent["x-time"], err)
}
if pattern := f.Wire.HeaderFormats["x-event-id"]; pattern != "" {
if !regexp.MustCompile(pattern).MatchString(sent["x-event-id"]) {
t.Errorf("x-event-id %q does not match %q", sent["x-event-id"], pattern)
}
}
// The origin the envelope claims is the one the bus enforces by namespace. A disagreement
// here means the envelope is lying about where it came from.
if sent["x-source"] != f.Given.Module {
t.Errorf("x-source is %q for module %q", sent["x-source"], f.Given.Module)
}
}
func TestTheGoSubjectMatchesTheFixture(t *testing.T) {
f := loadFixture(t, "events/module-event.json")
got := "mesh.mod." + f.Given.Module + ".event." + f.Given.Key
if got != f.Wire.Subject {
t.Errorf("this implementation would publish on %q; the fixture says %q", got, f.Wire.Subject)
}
}
internal/link/conformance_sources_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/conformance_test.go
-128
View File
@@ -1,12 +1,8 @@
package link
import (
"bytes"
"encoding/json"
"fmt"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strings"
@@ -61,25 +57,6 @@ var (
capturedLog = filepath.Join(beside.Captured(), "CAPTURED")
)
func loadFixture(t *testing.T, name string) fixture {
t.Helper()
raw, from, err := sdkFixture(os.Getenv(beside.Env), goModFile, capturedSDK, capturedLog, name)
if err != nil {
// Failed, never skipped: a skip here passed the suite with nothing judged.
t.Fatal(err)
}
t.Logf("%s: %s", name, from)
if strings.HasPrefix(from, notJudged) {
// Said on the check's own output too, where `go test` without -v prints no log of a passing test.
saidNotJudged.Do(func() { fmt.Fprintln(os.Stderr, "internal/link conformance: "+from) })
}
var f fixture
if err := json.Unmarshal(raw, &f); err != nil {
t.Fatalf("%s: %v", name, err)
}
return f
}
// notJudged opens what a merge check without the SDK's clone says; saidNotJudged says it once on stderr.
const notJudged = "NOT JUDGED AGAINST THE CLONE"
@@ -87,76 +64,9 @@ var saidNotJudged sync.Once
// sdkFixture is one conformance fixture and where it was read: from the mesh-sdk clone in root (a merge
// check's MESH_CHECK_BESIDE) at the pin of goMod, or, with no root, from the captured copy.
func sdkFixture(root, goMod, captured, capturedLog, name string) ([]byte, string, error) {
file := path.Join("conformance", name)
readCaptured := func() ([]byte, error) {
raw, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file)))
if err != nil {
return nil, fmt.Errorf("the captured SDK's %s: %w", file, err)
}
return raw, nil
}
if root == "" {
raw, err := readCaptured()
return raw, "the copy captured in testdata/beside (see CAPTURED); a merge check reads the SDK's clone " +
"at this tree's pin", err
}
clone := filepath.Join(root, "mesh-sdk")
if _, err := os.Stat(clone); err != nil {
// A build seat asked by a controller from before mesh-controller #220 places no mesh-sdk beside the
// check, so #220 could never pass its own check if this failed. Until #220 has rolled out, the
// captured copy is read and the log says so loudly; the follow-up of novox/hq issue 449 makes this
// a failure again.
at, cerr := capturedCommit(capturedLog)
if cerr != nil {
return nil, "", cerr
}
raw, rerr := readCaptured()
return raw, fmt.Sprintf(notJudged+": the seat placed no mesh-sdk beside this check in "+
"%s=%s (it does once mesh-controller #220 has rolled out); read the captured copy at %s", beside.Env,
root, at), rerr
}
ref, err := sdkRef(goMod)
if err != nil {
return nil, "", err
}
raw, err := gitShow(clone, ref, file)
if err != nil {
return nil, "", fmt.Errorf("the mesh-sdk clone beside this check has no %s at %s, the SDK this tree's "+
"go.mod pins: %w", file, ref, err)
}
// The captured copy is the SDK's own, never edited by hand: the clone at the captured commit says so.
at, err := capturedCommit(capturedLog)
if err != nil {
return nil, "", err
}
theirs, err := gitShow(clone, at, file)
if err != nil {
return nil, "", fmt.Errorf("the mesh-sdk clone has no %s at %s, the commit CAPTURED names: %w", file, at, err)
}
ours, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file)))
if err != nil {
return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err)
}
if !bytes.Equal(ours, theirs) {
return nil, "", fmt.Errorf("the captured %s is not mesh-sdk's at %s, the commit CAPTURED names: it was "+
"edited or captured wrong; capture it again as CAPTURED says", file, at)
}
return raw, "mesh-sdk cloned beside this check, at " + ref + " (this tree's go.mod)", nil
}
// gitShow is one file of a repository at a ref. safe.directory, because the clone is the build seat's and
// the test may run as another user.
func gitShow(repository, ref, file string) ([]byte, error) {
cmd := exec.Command("git", "-c", "safe.directory=*", "-C", repository, "show", ref+":"+file)
var stderr bytes.Buffer
cmd.Stderr = &stderr
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("git show %s:%s: %v: %s", ref, file, err, strings.TrimSpace(stderr.String()))
}
return out, nil
}
// pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970.
var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`)
@@ -216,50 +126,12 @@ func TestTheCapturedSDKIsTheOneGoModPins(t *testing.T) {
}
// Every header the fixture requires is one this implementation actually sets.
func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) {
f := loadFixture(t, "events/module-event.json")
sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node)
for _, want := range f.Wire.RequiredHeaders {
if _, ok := sent[want]; !ok {
t.Errorf("the Go emitter does not set %q, which the fixture requires — an event it "+
"emits is one a conforming consumer refuses", want)
}
}
}
// And each value is in the shape the fixture pins, because a header present but differently
// formatted is the disagreement that does not announce itself.
func TestTheGoEmittersHeaderFormatsMatch(t *testing.T) {
f := loadFixture(t, "events/module-event.json")
sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node)
if got := sent["content-type"]; got != f.Wire.HeaderFormats["content-type"] {
t.Errorf("content-type is %q, the fixture says %q", got, f.Wire.HeaderFormats["content-type"])
}
if _, err := time.Parse(time.RFC3339, sent["x-time"]); err != nil {
t.Errorf("x-time %q is not RFC3339, which the fixture requires: %v", sent["x-time"], err)
}
if pattern := f.Wire.HeaderFormats["x-event-id"]; pattern != "" {
if !regexp.MustCompile(pattern).MatchString(sent["x-event-id"]) {
t.Errorf("x-event-id %q does not match %q", sent["x-event-id"], pattern)
}
}
// The origin the envelope claims is the one the bus enforces by namespace. A disagreement
// here means the envelope is lying about where it came from.
if sent["x-source"] != f.Given.Module {
t.Errorf("x-source is %q for module %q", sent["x-source"], f.Given.Module)
}
}
// The subject a module's event lands on is derived, not carried — so this implementation must
// derive the same one the fixture names.
func TestTheGoSubjectMatchesTheFixture(t *testing.T) {
f := loadFixture(t, "events/module-event.json")
got := "mesh.mod." + f.Given.Module + ".event." + f.Given.Key
if got != f.Wire.Subject {
t.Errorf("this implementation would publish on %q; the fixture says %q", got, f.Wire.Subject)
}
}
// goEventHeaders is the header set EmitEvent produces, factored so conformance can see it
// without a broker. Kept beside the emitter so the two cannot drift apart silently.
internal/link/deadletters_ask_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/deadletters_integration_test.go
+279
View File
@@ -0,0 +1,279 @@
//go:build integration
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
func TestADeadLetterIsDeliveredAgainToItsConsumerAlone(t *testing.T) {
js := aBus(t)
keeping(t, js)
failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged")
handling := aModuleConsumer(t, js, "media", "radarr", "gitea.pull.merged")
const subject = "mesh.mod.gitea.event.pull.merged"
msg := &nats.Msg{Subject: subject, Data: []byte(`{"n":1}`), Header: nats.Header{}}
msg.Header.Set("x-node", "forge")
msg.Header.Set(nats.MsgIdHdr, "event-1")
if _, err := js.Context().PublishMsg(msg); err != nil {
t.Fatal(err)
}
if m := next(t, handling); m == nil {
t.Fatal("the other module was not handed the event")
} else {
_ = m.Ack()
}
d := givenUp(t, js, failing)
if d.Consumer != "media_sonarr" || d.Stream != "EVENTS" || d.Subject != subject || d.Deliveries != 5 ||
d.GaveUp.IsZero() || d.Published.IsZero() || d.Lost != "" {
t.Fatalf("kept as %+v", d)
}
held, err := HeldDeadLetters(js.Context())
if err != nil || held["EVENTS.media_sonarr"] != 1 {
t.Fatalf("held %v (%v)", held, err)
}
whole, err := DeadLetterNamed(js.Context(), d.ID)
if err != nil || whole.Body != `{"n":1}` || len(whole.Headers["x-node"]) != 1 || whole.Headers["x-node"][0] != "forge" {
t.Fatalf("one asked whole is %+v (%v)", whole, err)
}
_, to, err := DeliverAgain(js.Context(), d.ID)
if err != nil {
t.Fatal(err)
}
if to != "mesh.again.media_sonarr.mod.gitea.event.pull.merged" {
t.Fatalf("delivered again on %s", to)
}
again := next(t, failing)
if again == nil {
t.Fatal("the consumer that gave it up was not handed it again")
}
if string(again.Data()) != `{"n":1}` || again.Headers().Get("x-node") != "forge" ||
again.Headers().Get(AgainHeader) == "" {
t.Fatalf("handed again as %s %v", again.Data(), again.Headers())
}
if original, ok := broker.OriginalOfAgain(again.Subject()); !ok || original != subject {
t.Fatalf("delivered again on %s, which does not say the event's own subject", again.Subject())
}
_ = again.Ack()
if m := next(t, handling); m != nil {
t.Fatalf("the module that handled it was handed it twice: %s", m.Subject())
}
if _, err := DeadLetterNamed(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) {
t.Fatalf("still kept after it was delivered again: %v", err)
}
held, _ = HeldDeadLetters(js.Context())
if held["EVENTS.media_sonarr"] != 0 {
t.Fatalf("still held: %v", held)
}
// Asked twice, delivered once: the second finds nothing kept.
if _, _, err := DeliverAgain(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) {
t.Fatalf("a second delivery answered %v", err)
}
}
func TestADeadLetterDroppedIsGone(t *testing.T) {
js := aBus(t)
keeping(t, js)
failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged")
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil {
t.Fatal(err)
}
d := givenUp(t, js, failing)
if _, err := DropDeadLetter(js.Context(), d.ID); err != nil {
t.Fatal(err)
}
if left, total, err := DeadLetters(js.Context(), "", 0); err != nil || total != 0 || len(left) != 0 {
t.Fatalf("after the drop: %v %d %v", left, total, err)
}
if m := next(t, failing); m != nil {
t.Fatalf("a dropped event was handed over: %s", m.Subject())
}
}
func TestANoticeWhoseMessageIsGoneIsKeptAndSaysSo(t *testing.T) {
js := aBus(t)
d, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+
`"deliveries":5,"timestamp":"2026-10-06T15:51:00Z"}`))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(d.Lost, "no longer held message 42") || d.Consumer != "media_sonarr" {
t.Fatalf("kept as %+v", d)
}
if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "Drop it") {
t.Fatalf("a record without its message was delivered again: %v", err)
}
// The same notice taken twice is kept once.
if _, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+
`"deliveries":5}`)); err != nil {
t.Fatal(err)
}
if _, total, _ := DeadLetters(js.Context(), "media_sonarr", 0); total != 1 {
t.Fatalf("one notice taken twice is kept %d times", total)
}
}
func TestADeadLetterIsDeliveredAgainOnlyWhereItIsReceived(t *testing.T) {
js := aBus(t)
keeping(t, js)
failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged")
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil {
t.Fatal(err)
}
d := givenUp(t, js, failing)
// A consumer as it was before this fix: its filters do not take what is delivered again.
c, _ := broker.ConsumerFor(broker.Principal{Kind: broker.KindModule, Node: "media", Module: "sonarr",
Consumes: []string{"gitea.pull.merged"}, PasswordHash: "x"})
c.Filters = c.Filters[:len(c.Filters)-1]
if err := js.EnsureConsumer(c); err != nil {
t.Fatal(err)
}
if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "does not yet take") {
t.Fatalf("delivered to a consumer that does not filter its again subject: %v", err)
}
if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil {
t.Fatalf("a refused delivery let the dead letter go: %v", err)
}
// The consumer gone: nothing would receive it.
if err := js.Context().DeleteConsumer(broker.EventsStream, c.Name); err != nil {
t.Fatal(err)
}
if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "no longer on the bus") {
t.Fatalf("delivered to a consumer that is gone: %v", err)
}
if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil {
t.Fatalf("a refused delivery let the dead letter go: %v", err)
}
// Not an event: refused, whatever stream it is from.
for _, other := range []DeadLetter{
{ID: 2, Stream: "SEAT_TELEGRAM_SENDER", Consumer: "SEAT_TELEGRAM_SENDER_worker", Subject: "mesh.seat.telegram-sender.accept.send"},
{ID: 3, Stream: "KV_x", Consumer: "y", Subject: "$KV.x.k"},
} {
if _, err := AgainTo(js.Context(), other); err == nil {
t.Errorf("%s was given a subject to be delivered again on", other.Stream)
}
}
}
func TestTheListSaysTheTotalAndStopsAtItsLimit(t *testing.T) {
js := aBus(t)
for seq := 1; seq <= 5; seq++ {
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+
`"stream_seq":%d,"deliveries":5}`, seq))); err != nil {
t.Fatal(err)
}
}
list, total, err := DeadLetters(js.Context(), "media_sonarr", 2)
if err != nil || total != 5 || len(list) != 2 || list[0].Sequence != 5 {
t.Fatalf("%d of %d (%v): %+v", len(list), total, err, list)
}
if _, total, _ := DeadLetters(js.Context(), "another_one", 2); total != 0 {
t.Fatalf("another consumer's total is %d", total)
}
}
func TestTheControllerActsOnAnEventDeliveredAgain(t *testing.T) {
js := aBus(t)
told := &toldAbout{}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(told); err != nil {
t.Fatal(err)
}
if err := s.Answers(replaysWith{}); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
eventually(t, "the controller's event consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil
})
moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"})
if _, err := js.Context().Publish(broker.AgainSubject(broker.ControllerName, broker.ControllerFollows[0]), moved); err != nil {
t.Fatal(err)
}
eventually(t, "the upgrade delivered again reaching the controller", func() bool { return told.count() == 1 })
}
func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) {
js := aBus(t)
if err := js.Context().DeleteStream(broker.DeadLetterNoticesStream); err != nil {
t.Fatal(err)
}
stop := keepingGivenUp(context.Background(), js, quiet())
defer stop()
eventually(t, "the failure said", func() bool {
for _, a := range Advisories.Since(time.Now().Add(-time.Minute)) {
if a.Token == AdvisoryNotKept && a.Stream == broker.DeadLetterNoticesStream {
return true
}
}
return false
})
held := &counted{}
_, stopServing := servingOn(t, js, held)
defer stopServing()
if _, err := js.Context().Publish("mesh.control.anchor.report", []byte(`{"node":"anchor"}`)); err != nil {
t.Fatal(err)
}
eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 })
}
func TestTheNewestHeldDeadLetterIsWhenItWasKept(t *testing.T) {
js := aBus(t)
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
keep := func(consumer string, seq int, at time.Time) {
t.Helper()
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":%q,`+
`"stream_seq":%d,"deliveries":5,"timestamp":%q}`, consumer, seq, at.Format(time.RFC3339Nano)))); err != nil {
t.Fatal(err)
}
}
look := func() map[string]time.Time {
t.Helper()
held, err := HeldDeadLetters(js.Context())
if err != nil {
t.Fatal(err)
}
newest, err := NewestDeadLetters(js.Context(), held)
if err != nil {
t.Fatal(err)
}
return newest
}
// Sonarr gave up at 21:05 and that is kept first; radarr's letter is kept after it.
keep("media_sonarr", 2, gaveUp.Add(time.Minute))
keep("media_radarr", 3, gaveUp)
before := look()
if len(before) != 2 || before["EVENTS.media_sonarr"].IsZero() ||
!before["EVENTS.media_radarr"].After(before["EVENTS.media_sonarr"]) {
t.Fatalf("%v", before)
}
// Sonarr's give-up of 21:04, kept late, is newer than anything the condition has seen.
keep("media_sonarr", 1, gaveUp)
after := look()
if !after["EVENTS.media_sonarr"].After(before["EVENTS.media_sonarr"]) ||
!after["EVENTS.media_sonarr"].After(before["EVENTS.media_radarr"]) {
t.Fatalf("a letter kept late does not read as newer: before %v, after %v", before, after)
}
if !after["EVENTS.media_radarr"].Equal(before["EVENTS.media_radarr"]) {
t.Fatalf("radarr's newest moved: before %v, after %v", before, after)
}
}
internal/link/deadletters_module_stream_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/deadletters_restricted_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/deadletters_test.go
-261
View File
@@ -1,17 +1,12 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
@@ -85,117 +80,9 @@ func givenUp(t *testing.T, js *broker.JetStream, c jetstream.Consumer) DeadLette
// **Delivered again to the consumer that gave it up, and nobody else**: another module consuming the
// same event handled it the first time and must not handle it twice.
func TestADeadLetterIsDeliveredAgainToItsConsumerAlone(t *testing.T) {
js := aBus(t)
keeping(t, js)
failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged")
handling := aModuleConsumer(t, js, "media", "radarr", "gitea.pull.merged")
const subject = "mesh.mod.gitea.event.pull.merged"
msg := &nats.Msg{Subject: subject, Data: []byte(`{"n":1}`), Header: nats.Header{}}
msg.Header.Set("x-node", "forge")
msg.Header.Set(nats.MsgIdHdr, "event-1")
if _, err := js.Context().PublishMsg(msg); err != nil {
t.Fatal(err)
}
if m := next(t, handling); m == nil {
t.Fatal("the other module was not handed the event")
} else {
_ = m.Ack()
}
d := givenUp(t, js, failing)
if d.Consumer != "media_sonarr" || d.Stream != "EVENTS" || d.Subject != subject || d.Deliveries != 5 ||
d.GaveUp.IsZero() || d.Published.IsZero() || d.Lost != "" {
t.Fatalf("kept as %+v", d)
}
held, err := HeldDeadLetters(js.Context())
if err != nil || held["EVENTS.media_sonarr"] != 1 {
t.Fatalf("held %v (%v)", held, err)
}
whole, err := DeadLetterNamed(js.Context(), d.ID)
if err != nil || whole.Body != `{"n":1}` || len(whole.Headers["x-node"]) != 1 || whole.Headers["x-node"][0] != "forge" {
t.Fatalf("one asked whole is %+v (%v)", whole, err)
}
_, to, err := DeliverAgain(js.Context(), d.ID)
if err != nil {
t.Fatal(err)
}
if to != "mesh.again.media_sonarr.mod.gitea.event.pull.merged" {
t.Fatalf("delivered again on %s", to)
}
again := next(t, failing)
if again == nil {
t.Fatal("the consumer that gave it up was not handed it again")
}
if string(again.Data()) != `{"n":1}` || again.Headers().Get("x-node") != "forge" ||
again.Headers().Get(AgainHeader) == "" {
t.Fatalf("handed again as %s %v", again.Data(), again.Headers())
}
if original, ok := broker.OriginalOfAgain(again.Subject()); !ok || original != subject {
t.Fatalf("delivered again on %s, which does not say the event's own subject", again.Subject())
}
_ = again.Ack()
if m := next(t, handling); m != nil {
t.Fatalf("the module that handled it was handed it twice: %s", m.Subject())
}
if _, err := DeadLetterNamed(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) {
t.Fatalf("still kept after it was delivered again: %v", err)
}
held, _ = HeldDeadLetters(js.Context())
if held["EVENTS.media_sonarr"] != 0 {
t.Fatalf("still held: %v", held)
}
// Asked twice, delivered once: the second finds nothing kept.
if _, _, err := DeliverAgain(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) {
t.Fatalf("a second delivery answered %v", err)
}
}
func TestADeadLetterDroppedIsGone(t *testing.T) {
js := aBus(t)
keeping(t, js)
failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged")
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil {
t.Fatal(err)
}
d := givenUp(t, js, failing)
if _, err := DropDeadLetter(js.Context(), d.ID); err != nil {
t.Fatal(err)
}
if left, total, err := DeadLetters(js.Context(), "", 0); err != nil || total != 0 || len(left) != 0 {
t.Fatalf("after the drop: %v %d %v", left, total, err)
}
if m := next(t, failing); m != nil {
t.Fatalf("a dropped event was handed over: %s", m.Subject())
}
}
// A notice whose message its stream no longer holds is kept as a record that says so — said, and
// dropped by a person, never silently missing — and cannot be delivered again.
func TestANoticeWhoseMessageIsGoneIsKeptAndSaysSo(t *testing.T) {
js := aBus(t)
d, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+
`"deliveries":5,"timestamp":"2026-10-06T15:51:00Z"}`))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(d.Lost, "no longer held message 42") || d.Consumer != "media_sonarr" {
t.Fatalf("kept as %+v", d)
}
if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "Drop it") {
t.Fatalf("a record without its message was delivered again: %v", err)
}
// The same notice taken twice is kept once.
if _, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+
`"deliveries":5}`)); err != nil {
t.Fatal(err)
}
if _, total, _ := DeadLetters(js.Context(), "media_sonarr", 0); total != 1 {
t.Fatalf("one notice taken twice is kept %d times", total)
}
}
// What only the giving-up consumer filters is its own: a subject delivered again reads back as the
// event's, and a module's runtime reads the same key from it (the tokens around `.event.`).
@@ -221,163 +108,15 @@ func TestASubjectDeliveredAgainSaysTheEventsOwn(t *testing.T) {
// Only an event is delivered again, and only to a consumer that is on the bus and filters its again
// subject: a dead letter is never let go as delivered while nobody receives it.
func TestADeadLetterIsDeliveredAgainOnlyWhereItIsReceived(t *testing.T) {
js := aBus(t)
keeping(t, js)
failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged")
if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil {
t.Fatal(err)
}
d := givenUp(t, js, failing)
// A consumer as it was before this fix: its filters do not take what is delivered again.
c, _ := broker.ConsumerFor(broker.Principal{Kind: broker.KindModule, Node: "media", Module: "sonarr",
Consumes: []string{"gitea.pull.merged"}, PasswordHash: "x"})
c.Filters = c.Filters[:len(c.Filters)-1]
if err := js.EnsureConsumer(c); err != nil {
t.Fatal(err)
}
if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "does not yet take") {
t.Fatalf("delivered to a consumer that does not filter its again subject: %v", err)
}
if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil {
t.Fatalf("a refused delivery let the dead letter go: %v", err)
}
// The consumer gone: nothing would receive it.
if err := js.Context().DeleteConsumer(broker.EventsStream, c.Name); err != nil {
t.Fatal(err)
}
if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "no longer on the bus") {
t.Fatalf("delivered to a consumer that is gone: %v", err)
}
if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil {
t.Fatalf("a refused delivery let the dead letter go: %v", err)
}
// Not an event: refused, whatever stream it is from.
for _, other := range []DeadLetter{
{ID: 2, Stream: "SEAT_TELEGRAM_SENDER", Consumer: "SEAT_TELEGRAM_SENDER_worker", Subject: "mesh.seat.telegram-sender.accept.send"},
{ID: 3, Stream: "KV_x", Consumer: "y", Subject: "$KV.x.k"},
} {
if _, err := AgainTo(js.Context(), other); err == nil {
t.Errorf("%s was given a subject to be delivered again on", other.Stream)
}
}
}
// A total says how many are held, however few the list carries.
func TestTheListSaysTheTotalAndStopsAtItsLimit(t *testing.T) {
js := aBus(t)
for seq := 1; seq <= 5; seq++ {
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+
`"stream_seq":%d,"deliveries":5}`, seq))); err != nil {
t.Fatal(err)
}
}
list, total, err := DeadLetters(js.Context(), "media_sonarr", 2)
if err != nil || total != 5 || len(list) != 2 || list[0].Sequence != 5 {
t.Fatalf("%d of %d (%v): %+v", len(list), total, err, list)
}
if _, total, _ := DeadLetters(js.Context(), "another_one", 2); total != 0 {
t.Fatalf("another consumer's total is %d", total)
}
}
// An event the controller itself gave up on, delivered again by a person, is acted on as the event it
// was: it arrives under the controller's again subject, which its consumer filters.
func TestTheControllerActsOnAnEventDeliveredAgain(t *testing.T) {
js := aBus(t)
told := &toldAbout{}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(told); err != nil {
t.Fatal(err)
}
if err := s.Answers(replaysWith{}); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
eventually(t, "the controller's event consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil
})
moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"})
if _, err := js.Context().Publish(broker.AgainSubject(broker.ControllerName, broker.ControllerFollows[0]), moved); err != nil {
t.Fatal(err)
}
eventually(t, "the upgrade delivered again reaching the controller", func() bool { return told.count() == 1 })
}
// The notices cannot be taken (their stream is gone): said as a condition, and the controller serves on.
func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) {
js := aBus(t)
if err := js.Context().DeleteStream(broker.DeadLetterNoticesStream); err != nil {
t.Fatal(err)
}
stop := keepingGivenUp(context.Background(), js, quiet())
defer stop()
eventually(t, "the failure said", func() bool {
for _, a := range Advisories.Since(time.Now().Add(-time.Minute)) {
if a.Token == AdvisoryNotKept && a.Stream == broker.DeadLetterNoticesStream {
return true
}
}
return false
})
held := &counted{}
_, stopServing := servingOn(t, js, held)
defer stopServing()
if _, err := js.Context().Publish("mesh.control.anchor.report", []byte(`{"node":"anchor"}`)); err != nil {
t.Fatal(err)
}
eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 })
}
// When DEAD_LETTERS kept each consumer's newest message, for the condition that counts what was given up
// on rather than how often it was looked at (novox/hq issue 440). It rises with every message kept, even
// one the consumer gave up on before the last: a notice that could not be kept is offered again a minute
// later, so a later give-up can be kept first, and the one kept after it must still count.
func TestTheNewestHeldDeadLetterIsWhenItWasKept(t *testing.T) {
js := aBus(t)
gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC)
keep := func(consumer string, seq int, at time.Time) {
t.Helper()
if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":%q,`+
`"stream_seq":%d,"deliveries":5,"timestamp":%q}`, consumer, seq, at.Format(time.RFC3339Nano)))); err != nil {
t.Fatal(err)
}
}
look := func() map[string]time.Time {
t.Helper()
held, err := HeldDeadLetters(js.Context())
if err != nil {
t.Fatal(err)
}
newest, err := NewestDeadLetters(js.Context(), held)
if err != nil {
t.Fatal(err)
}
return newest
}
// Sonarr gave up at 21:05 and that is kept first; radarr's letter is kept after it.
keep("media_sonarr", 2, gaveUp.Add(time.Minute))
keep("media_radarr", 3, gaveUp)
before := look()
if len(before) != 2 || before["EVENTS.media_sonarr"].IsZero() ||
!before["EVENTS.media_radarr"].After(before["EVENTS.media_sonarr"]) {
t.Fatalf("%v", before)
}
// Sonarr's give-up of 21:04, kept late, is newer than anything the condition has seen.
keep("media_sonarr", 1, gaveUp)
after := look()
if !after["EVENTS.media_sonarr"].After(before["EVENTS.media_sonarr"]) ||
!after["EVENTS.media_sonarr"].After(before["EVENTS.media_radarr"]) {
t.Fatalf("a letter kept late does not read as newer: before %v, after %v", before, after)
}
if !after["EVENTS.media_radarr"].Equal(before["EVENTS.media_radarr"]) {
t.Fatalf("radarr's newest moved: before %v, after %v", before, after)
}
}
internal/link/enrol_bus_credential_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link_test
import (
internal/link/enrol_claim_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link_test
import (
internal/link/enrol_reply_probe_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/enrol_shape_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link_test
import (
internal/link/enrol_tunnel_key_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link_test
import (
internal/link/handacts_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/handover_integration_test.go
+75
View File
@@ -0,0 +1,75 @@
//go:build integration
package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
func TestAHandOverIsAskedOfTheMachineAndItsAnswerComesBack(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
signer, public := testSigner(t)
var heard HandOverAsk
engine, err := conn.Subscribe(broker.AskHandOverSubject("laptop"), func(msg *nats.Msg) {
var signed SignedHandOver
_ = json.Unmarshal(msg.Data, &signed)
if !ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) {
_ = msg.Respond([]byte(`{"refused":"not the mesh's signature"}`))
return
}
_ = json.Unmarshal(signed.Ask, &heard)
switch heard.Path {
case "/srv/notes":
body, _ := json.Marshal(HandOverAnswer{Said: "/srv/notes (notes.data) is handed to the mesh by " + heard.By})
_ = msg.Respond(body)
case "/srv/empty":
_ = msg.Respond([]byte(`{}`))
default:
body, _ := json.Marshal(HandOverAnswer{Refused: heard.Path + " is not a directory this machine uses as found; nothing was handed over"})
_ = msg.Respond(body)
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = engine.Unsubscribe() })
ctx := context.Background()
a, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/notes", "jo through mesh-cli on anchor", 5*time.Second)
if err != nil || a.Refused != "" || !strings.Contains(a.Said, "handed to the mesh by jo through mesh-cli on anchor") {
t.Fatalf("answered %+v, %v", a, err)
}
if heard.Node != "laptop" || heard.Path != "/srv/notes" || heard.By != "jo through mesh-cli on anchor" {
t.Fatalf("the engine heard %+v", heard)
}
a, err = AskHandOver(ctx, conn, signer, "laptop", "/srv/other", "jo", 5*time.Second)
if err != nil || a.Said != "" || !strings.Contains(a.Refused, "nothing was handed over") {
t.Fatalf("a refusal came back as %+v, %v", a, err)
}
if _, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/empty", "jo", 5*time.Second); err == nil ||
!strings.Contains(err.Error(), "neither") {
t.Fatalf("an empty answer was taken: %v", err)
}
if _, err := AskHandOver(ctx, conn, signer, "anchor", "/srv/notes", "jo", 5*time.Second); err == nil ||
!strings.Contains(err.Error(), "node-engine") {
t.Fatalf("a machine with no engine listening: %v", err)
}
if _, err := AskHandOver(ctx, nil, signer, "anchor", "/srv/notes", "jo", time.Second); err == nil {
t.Fatal("asked with no bus")
}
}
internal/link/handover_test.go
-62
View File
@@ -4,14 +4,10 @@ import (
"context"
"crypto/ed25519"
"encoding/json"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
// The hand-over's ask and answer hold these field names; the engine's side holds the same list (mesh-host
@@ -99,64 +95,6 @@ func TestAHandOverIsSignedWithAContextANonceAndAnExpiry(t *testing.T) {
// The ask reaches the machine's engine on its own subject and its answer comes back whole: what it recorded, or
// its refusal as the engine worded it. A machine with no engine listening is said as not answering, and an
// answer that is neither is refused rather than read as a record.
func TestAHandOverIsAskedOfTheMachineAndItsAnswerComesBack(t *testing.T) {
url := testbus.URL(t)
conn, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
signer, public := testSigner(t)
var heard HandOverAsk
engine, err := conn.Subscribe(broker.AskHandOverSubject("laptop"), func(msg *nats.Msg) {
var signed SignedHandOver
_ = json.Unmarshal(msg.Data, &signed)
if !ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) {
_ = msg.Respond([]byte(`{"refused":"not the mesh's signature"}`))
return
}
_ = json.Unmarshal(signed.Ask, &heard)
switch heard.Path {
case "/srv/notes":
body, _ := json.Marshal(HandOverAnswer{Said: "/srv/notes (notes.data) is handed to the mesh by " + heard.By})
_ = msg.Respond(body)
case "/srv/empty":
_ = msg.Respond([]byte(`{}`))
default:
body, _ := json.Marshal(HandOverAnswer{Refused: heard.Path + " is not a directory this machine uses as found; nothing was handed over"})
_ = msg.Respond(body)
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = engine.Unsubscribe() })
ctx := context.Background()
a, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/notes", "jo through mesh-cli on anchor", 5*time.Second)
if err != nil || a.Refused != "" || !strings.Contains(a.Said, "handed to the mesh by jo through mesh-cli on anchor") {
t.Fatalf("answered %+v, %v", a, err)
}
if heard.Node != "laptop" || heard.Path != "/srv/notes" || heard.By != "jo through mesh-cli on anchor" {
t.Fatalf("the engine heard %+v", heard)
}
a, err = AskHandOver(ctx, conn, signer, "laptop", "/srv/other", "jo", 5*time.Second)
if err != nil || a.Said != "" || !strings.Contains(a.Refused, "nothing was handed over") {
t.Fatalf("a refusal came back as %+v, %v", a, err)
}
if _, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/empty", "jo", 5*time.Second); err == nil ||
!strings.Contains(err.Error(), "neither") {
t.Fatalf("an empty answer was taken: %v", err)
}
if _, err := AskHandOver(ctx, conn, signer, "anchor", "/srv/notes", "jo", 5*time.Second); err == nil ||
!strings.Contains(err.Error(), "node-engine") {
t.Fatalf("a machine with no engine listening: %v", err)
}
if _, err := AskHandOver(ctx, nil, signer, "anchor", "/srv/notes", "jo", time.Second); err == nil {
t.Fatal("asked with no bus")
}
}
// A machine's grant hears its own hand-over ask and nobody else's, and may answer it: the one request a node is
// asked (novox/hq issue 356).
internal/link/heard_order_integration_test.go
+118
View File
@@ -0,0 +1,118 @@
//go:build integration
package link_test
import (
"context"
"testing"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) {
t.Helper()
inv := inventory.ForTest(t)
node, err := inv.AddNode(context.Background(), "home-server")
if err != nil {
t.Fatal(err)
}
return inv, link.Enrolment{Inventory: inv}, node.ID
}
func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) {
inv, heard, id := anOrderedMachine(t)
ctx := context.Background()
before := countFor(link.WriterNodeEngine("home-server"))
if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil {
t.Fatal(err)
}
doing, _, err := inv.DoingOf(ctx, "home-server")
if err != nil || doing.Declared != "d12" || doing.Applied != 2 {
t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err)
}
if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 {
t.Fatalf("the refusal was counted %d times, want once", got)
}
kept, err := inv.KeptOrder(ctx, id)
if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) {
t.Fatalf("the order kept is %+v (%v)", kept, err)
}
}
func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) {
inv, heard, _ := anOrderedMachine(t)
ctx := context.Background()
if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
t.Fatal(err)
}
failed := ordered("d12", 57, 12, 40)
failed.Failed = map[string]string{"b": "it failed a moment before it applied"}
if _, err := heard.Heard(ctx, failed); err != nil {
t.Fatal(err)
}
doing, _, _ := inv.DoingOf(ctx, "home-server")
if doing.Outcome != inventory.OutcomeApplied {
t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing)
}
// A newer report of it — the next reconcile — is kept.
again := ordered("d12", 57, 12, 42)
again.Failed = map[string]string{"b": "it failed since"}
if _, err := heard.Heard(ctx, again); err != nil {
t.Fatal(err)
}
if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed {
t.Fatalf("a newer report of the same declaration was not kept: %+v", doing)
}
}
func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) {
inv, heard, id := anOrderedMachine(t)
ctx := context.Background()
if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
t.Fatal(err)
}
if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" {
t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing)
}
// And the machine reads an epoch: it orders its reports.
if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads {
t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err)
}
}
func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) {
inv, heard, id := anOrderedMachine(t)
ctx := context.Background()
if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil {
t.Fatal(err)
}
if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" {
t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing)
}
if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) {
t.Fatalf("an unordered account left the order kept: %+v", kept)
}
if reads, _ := inv.ReadsEpoch(ctx, id); reads {
t.Fatal("a node-engine that orders nothing is still sent an epoch")
}
// The next ordered account, whatever its numbers, has nothing to be older than.
if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil {
t.Fatal(err)
}
if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" {
t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing)
}
}
internal/link/heard_order_test.go
-106
View File
@@ -1,11 +1,9 @@
package link_test
import (
"context"
"testing"
"time"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -14,122 +12,18 @@ import (
// Of the same declaration, report r then r−1: refused. The digest the mesh last sent decides nothing
// for an ordered report; an unordered one, from an older node-engine, is judged by it as before.
func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) {
t.Helper()
inv := inventory.ForTest(t)
node, err := inv.AddNode(context.Background(), "home-server")
if err != nil {
t.Fatal(err)
}
return inv, link.Enrolment{Inventory: inv}, node.ID
}
func ordered(declared string, epoch, sequence, report int64, applied ...string) link.Report {
return link.Report{Node: "home-server", Declared: declared, Applied: applied,
Order: link.Order{Epoch: epoch, Sequence: sequence}, ReportSequence: report}
}
// Issue 267, ordered: a reconcile's account of declaration 11 reaches the mesh after the apply's of 12.
func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) {
inv, heard, id := anOrderedMachine(t)
ctx := context.Background()
before := countFor(link.WriterNodeEngine("home-server"))
if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil {
t.Fatal(err)
}
doing, _, err := inv.DoingOf(ctx, "home-server")
if err != nil || doing.Declared != "d12" || doing.Applied != 2 {
t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err)
}
if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 {
t.Fatalf("the refusal was counted %d times, want once", got)
}
kept, err := inv.KeptOrder(ctx, id)
if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) {
t.Fatalf("the order kept is %+v (%v)", kept, err)
}
}
func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) {
inv, heard, _ := anOrderedMachine(t)
ctx := context.Background()
if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil {
t.Fatal(err)
}
failed := ordered("d12", 57, 12, 40)
failed.Failed = map[string]string{"b": "it failed a moment before it applied"}
if _, err := heard.Heard(ctx, failed); err != nil {
t.Fatal(err)
}
doing, _, _ := inv.DoingOf(ctx, "home-server")
if doing.Outcome != inventory.OutcomeApplied {
t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing)
}
// A newer report of it — the next reconcile — is kept.
again := ordered("d12", 57, 12, 42)
again.Failed = map[string]string{"b": "it failed since"}
if _, err := heard.Heard(ctx, again); err != nil {
t.Fatal(err)
}
if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed {
t.Fatalf("a newer report of the same declaration was not kept: %+v", doing)
}
}
// An ordered account is judged by its order, not by the digest the mesh last sent: the account of 12
// is kept although the mesh has sent 13 since — it is still the newest account of the machine.
func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) {
inv, heard, id := anOrderedMachine(t)
ctx := context.Background()
if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
t.Fatal(err)
}
if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" {
t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing)
}
// And the machine reads an epoch: it orders its reports.
if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads {
t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err)
}
}
// A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the
// order kept is cleared, and it is no longer sent an epoch.
func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) {
inv, heard, id := anOrderedMachine(t)
ctx := context.Background()
if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil {
t.Fatal(err)
}
if err := inv.RecordSent(ctx, id, "d13", nil); err != nil {
t.Fatal(err)
}
if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil {
t.Fatal(err)
}
if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" {
t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing)
}
if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) {
t.Fatalf("an unordered account left the order kept: %+v", kept)
}
if reads, _ := inv.ReadsEpoch(ctx, id); reads {
t.Fatal("a node-engine that orders nothing is still sent an epoch")
}
// The next ordered account, whatever its numbers, has nothing to be older than.
if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil {
t.Fatal(err)
}
if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" {
t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing)
}
}
// A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose
// own report was lost is still counted from the machine's own tally.
internal/link/heard_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link_test
import (
internal/link/judge_directories_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/meshcli_integration_test.go
+96
View File
@@ -0,0 +1,96 @@
//go:build integration
package link
import (
"context"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
func cliBus(t *testing.T, l *CallLog, atOnce int, handle CLIHandler) *nats.Conn {
t.Helper()
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
stop, err := OverNATS{Conn: conn}.serveCLI(l, atOnce, handle, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(stop)
return conn
}
func TestALongMeshCLILineIsFollowedToItsAnswer(t *testing.T) {
was := AnswerWithin
AnswerWithin = 50 * time.Millisecond
t.Cleanup(func() { AnswerWithin = was })
release := make(chan struct{})
conn := cliBus(t, NewCallLog(), 4, func(ctx context.Context, node string, asked CLIAsked) CLIAnswer {
<-release
return CLIAnswer{Stdout: []byte("done on " + node), Terminal: true}
})
_, first, failed := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000})
call, _ := first["call"].(string)
if failed != "" || first["running"] != true || call == "" {
t.Fatalf("a long line was not answered as running with its call: %v %q", first, failed)
}
_, still, _ := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000})
if still["running"] != true {
t.Fatalf("following a running line answered %v", still)
}
close(release)
deadline := time.Now().Add(5 * time.Second)
for {
a, raw, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000})
if failed != "" {
t.Fatalf("following answered %q", failed)
}
if raw["running"] != true {
if string(a.Stdout) != "done on laptop" || !a.Terminal {
t.Fatalf("followed to %+v", a)
}
break
}
if time.Now().After(deadline) {
t.Fatal("the line never finished for its follower")
}
time.Sleep(20 * time.Millisecond)
}
if _, _, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "agent", UID: 1001}); failed == "" {
t.Fatal("another account followed the operator's line")
}
if _, _, failed := askCLI(t, conn, "desktop", CLIAsked{Follow: call, Account: "op", UID: 1000}); failed == "" {
t.Fatal("another node followed the line")
}
}
func TestMeshCLILinesAtOnceAreBounded(t *testing.T) {
was := AnswerWithin
AnswerWithin = 50 * time.Millisecond
t.Cleanup(func() { AnswerWithin = was })
release := make(chan struct{})
t.Cleanup(func() { close(release) })
ran := make(chan struct{}, 4)
conn := cliBus(t, NewCallLog(), 1, func(context.Context, string, CLIAsked) CLIAnswer {
ran <- struct{}{}
<-release
return CLIAnswer{}
})
if _, first, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}); first["running"] != true {
t.Fatalf("the first line answered %v", first)
}
a, _, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"})
if !strings.Contains(a.Refused, "busy") || a.Exit == 0 {
t.Fatalf("a line over the bound answered %+v", a)
}
if len(ran) != 1 {
t.Fatalf("%d lines ran, one was bound", len(ran))
}
}
internal/link/meshcli_test.go
-82
View File
@@ -10,8 +10,6 @@ import (
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// The node-engine's request and the answer it hands mesh-cli hold these field names; the engine's side holds the
@@ -94,20 +92,6 @@ func TestALargeMeshCLIAnswerIsCutAndSaysSo(t *testing.T) {
}
// cliBus serves mesh-cli on a bus of the test's own with a call log of its own, and returns a connection to ask on.
func cliBus(t *testing.T, l *CallLog, atOnce int, handle CLIHandler) *nats.Conn {
t.Helper()
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
stop, err := OverNATS{Conn: conn}.serveCLI(l, atOnce, handle, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(stop)
return conn
}
// askCLI asks one request on a node's subject and reads the envelope.
func askCLI(t *testing.T, conn *nats.Conn, node string, asked CLIAsked) (CLIAnswer, map[string]any, string) {
@@ -134,74 +118,8 @@ func askCLI(t *testing.T, conn *nats.Conn, node string, asked CLIAsked) (CLIAnsw
// **A line that outlasts the bus's window for an answer is followed to its answer, never lost** (review of ADR
// 0272): the first answer says it is running and names its call, and following the call by the same account on
// the same node gives what the line printed once it ends. Another account, or another node, is told no such call.
func TestALongMeshCLILineIsFollowedToItsAnswer(t *testing.T) {
was := AnswerWithin
AnswerWithin = 50 * time.Millisecond
t.Cleanup(func() { AnswerWithin = was })
release := make(chan struct{})
conn := cliBus(t, NewCallLog(), 4, func(ctx context.Context, node string, asked CLIAsked) CLIAnswer {
<-release
return CLIAnswer{Stdout: []byte("done on " + node), Terminal: true}
})
_, first, failed := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000})
call, _ := first["call"].(string)
if failed != "" || first["running"] != true || call == "" {
t.Fatalf("a long line was not answered as running with its call: %v %q", first, failed)
}
_, still, _ := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000})
if still["running"] != true {
t.Fatalf("following a running line answered %v", still)
}
close(release)
deadline := time.Now().Add(5 * time.Second)
for {
a, raw, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000})
if failed != "" {
t.Fatalf("following answered %q", failed)
}
if raw["running"] != true {
if string(a.Stdout) != "done on laptop" || !a.Terminal {
t.Fatalf("followed to %+v", a)
}
break
}
if time.Now().After(deadline) {
t.Fatal("the line never finished for its follower")
}
time.Sleep(20 * time.Millisecond)
}
if _, _, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "agent", UID: 1001}); failed == "" {
t.Fatal("another account followed the operator's line")
}
if _, _, failed := askCLI(t, conn, "desktop", CLIAsked{Follow: call, Account: "op", UID: 1000}); failed == "" {
t.Fatal("another node followed the line")
}
}
// Lines running at once are bounded: one over the bound is answered busy at once, and nothing ran.
func TestMeshCLILinesAtOnceAreBounded(t *testing.T) {
was := AnswerWithin
AnswerWithin = 50 * time.Millisecond
t.Cleanup(func() { AnswerWithin = was })
release := make(chan struct{})
t.Cleanup(func() { close(release) })
ran := make(chan struct{}, 4)
conn := cliBus(t, NewCallLog(), 1, func(context.Context, string, CLIAsked) CLIAnswer {
ran <- struct{}{}
<-release
return CLIAnswer{}
})
if _, first, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}); first["running"] != true {
t.Fatalf("the first line answered %v", first)
}
a, _, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"})
if !strings.Contains(a.Refused, "busy") || a.Exit == 0 {
t.Fatalf("a line over the bound answered %+v", a)
}
if len(ran) != 1 {
t.Fatalf("%d lines ran, one was bound", len(ran))
}
}
// A mesh-cli line's record keeps its first word, never the rest of its line, and its answer is never kept on the
// bus, where `calls` answers anyone who may call the seat.
internal/link/movesecret_integration_test.go
+58
View File
@@ -0,0 +1,58 @@
//go:build integration
package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
func TestAMoveIsAskedOfTheMachineSignedUnderItsOwnContext(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
signer, public := testSigner(t)
var heard HandOverAsk
var row MoveSecretAsk
engine, err := conn.Subscribe(broker.AskMoveSecretSubject("laptop"), func(msg *nats.Msg) {
var signed SignedHandOver
_ = json.Unmarshal(msg.Data, &signed)
if ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) ||
!ed25519.Verify(public, append([]byte(MoveSecretContext), signed.Ask...), signed.Signature) {
_ = msg.Respond([]byte(`{"refused":"not signed as a move"}`))
return
}
_ = json.Unmarshal(signed.Ask, &heard)
_ = json.Unmarshal(signed.Ask, &row)
body, _ := json.Marshal(MoveSecretAnswer{Said: "moved", Secret: "own/laptop/" + row.Module + "/" + row.Name,
Generation: 1})
_ = msg.Respond(body)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = engine.Unsubscribe() })
a, err := AskMoveSecret(context.Background(), conn, signer, "laptop", "the move",
MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2Vh"}, 5*time.Second)
if err != nil || a.Secret != "own/laptop/postgres/superuser-password" || a.Generation != 1 {
t.Fatalf("answered %+v, %v", a, err)
}
if heard.Node != "laptop" || heard.Path != "" || heard.Nonce == "" || heard.Expires.IsZero() || heard.By != "the move" ||
row.Sealed != "c2Vh" || row.Origin != "made" {
t.Fatalf("the engine heard %+v %+v", heard, row)
}
if _, err := AskMoveSecret(context.Background(), conn, signer, "anchor", "the move", MoveSecretAsk{}, 2*time.Second); err == nil ||
!strings.Contains(err.Error(), "nothing was moved") {
t.Fatalf("a machine with no engine listening: %v", err)
}
}
internal/link/movesecret_test.go
-46
View File
@@ -1,18 +1,13 @@
package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"sort"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
// The move's ask and answer keep the engine's field names (mesh-host internal/link,
@@ -47,44 +42,3 @@ func TestTheMovesAskAndAnswerKeepTheirFieldNames(t *testing.T) {
// A move is asked of the machine on its own subject, signed under its own context — never a hand-over's — naming
// no path and carrying the row beside the hand-over's fields; the engine's answer comes back whole.
func TestAMoveIsAskedOfTheMachineSignedUnderItsOwnContext(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(conn.Close)
signer, public := testSigner(t)
var heard HandOverAsk
var row MoveSecretAsk
engine, err := conn.Subscribe(broker.AskMoveSecretSubject("laptop"), func(msg *nats.Msg) {
var signed SignedHandOver
_ = json.Unmarshal(msg.Data, &signed)
if ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) ||
!ed25519.Verify(public, append([]byte(MoveSecretContext), signed.Ask...), signed.Signature) {
_ = msg.Respond([]byte(`{"refused":"not signed as a move"}`))
return
}
_ = json.Unmarshal(signed.Ask, &heard)
_ = json.Unmarshal(signed.Ask, &row)
body, _ := json.Marshal(MoveSecretAnswer{Said: "moved", Secret: "own/laptop/" + row.Module + "/" + row.Name,
Generation: 1})
_ = msg.Respond(body)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = engine.Unsubscribe() })
a, err := AskMoveSecret(context.Background(), conn, signer, "laptop", "the move",
MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2Vh"}, 5*time.Second)
if err != nil || a.Secret != "own/laptop/postgres/superuser-password" || a.Generation != 1 {
t.Fatalf("answered %+v, %v", a, err)
}
if heard.Node != "laptop" || heard.Path != "" || heard.Nonce == "" || heard.Expires.IsZero() || heard.By != "the move" ||
row.Sealed != "c2Vh" || row.Origin != "made" {
t.Fatalf("the engine heard %+v %+v", heard, row)
}
if _, err := AskMoveSecret(context.Background(), conn, signer, "anchor", "the move", MoveSecretAsk{}, 2*time.Second); err == nil ||
!strings.Contains(err.Error(), "nothing was moved") {
t.Fatalf("a machine with no engine listening: %v", err)
}
}
internal/link/pages_integration_test.go
+59
View File
@@ -0,0 +1,59 @@
//go:build integration
package link
import (
"context"
"encoding/json"
"strings"
"testing"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
func TestAPartGoesOnlyToItsCaller(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
l := NewCallLog()
c := l.begin("mesh-controller", "status", nil, "_INBOX.operator.ABCDEFGHIJKLMNOPQRSTUV")
l.page(c, []byte(strings.Repeat("z", 5000)), 2048)
sub, err := conn.Subscribe("pages.test", func(m *nats.Msg) { l.servePage("mesh-controller", "status", m, nil) })
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe()
_, err = askPart(context.Background(), conn, "pages.test", Paged{Call: c.ID, Parts: 3}, 0)
if err == nil || !strings.Contains(err.Error(), "another caller") {
t.Fatalf("a part was handed to a caller it was not paged to: %v", err)
}
_, err = askPart(context.Background(), conn, "pages.test", Paged{Call: "call-0-0", Parts: 3}, 0)
if err == nil || !strings.Contains(err.Error(), "not held here") {
t.Fatalf("a part of nothing held: %v", err)
}
}
func TestAMissingPartFailsTheWhole(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
sub, err := conn.Subscribe("pages.none", func(m *nats.Msg) { _ = m.Respond([]byte(`{"error":"gone"}`)) })
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe()
first, _ := json.Marshal(map[string]any{"error": "too large", "paged": Paged{Call: "call-1-1", Bytes: 10, Parts: 2}})
_, err = Whole(context.Background(), conn, "pages.none", first)
if err == nil || !strings.Contains(err.Error(), "part 0") || !strings.Contains(err.Error(), "gone") {
t.Fatalf("a missing part read as %v", err)
}
if plain, err := Whole(context.Background(), conn, "pages.none", []byte(`{"result":1}`)); err != nil ||
string(plain) != `{"result":1}` {
t.Fatalf("an answer that fits was changed: %s %v", plain, err)
}
}
internal/link/pages_test.go
-46
View File
@@ -7,8 +7,6 @@ import (
"testing"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// An answer too large for one message is recorded as paged, and a call's record says so: `calls` no
@@ -53,49 +51,5 @@ func TestAnAnswerTheBusRefusesIsSaid(t *testing.T) {
}
// A part goes only to the caller its answer was paged to, on the verb that answered it.
func TestAPartGoesOnlyToItsCaller(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
l := NewCallLog()
c := l.begin("mesh-controller", "status", nil, "_INBOX.operator.ABCDEFGHIJKLMNOPQRSTUV")
l.page(c, []byte(strings.Repeat("z", 5000)), 2048)
sub, err := conn.Subscribe("pages.test", func(m *nats.Msg) { l.servePage("mesh-controller", "status", m, nil) })
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe()
_, err = askPart(context.Background(), conn, "pages.test", Paged{Call: c.ID, Parts: 3}, 0)
if err == nil || !strings.Contains(err.Error(), "another caller") {
t.Fatalf("a part was handed to a caller it was not paged to: %v", err)
}
_, err = askPart(context.Background(), conn, "pages.test", Paged{Call: "call-0-0", Parts: 3}, 0)
if err == nil || !strings.Contains(err.Error(), "not held here") {
t.Fatalf("a part of nothing held: %v", err)
}
}
// A part that cannot be had fails the whole answer, naming it — never a shorter answer.
func TestAMissingPartFailsTheWhole(t *testing.T) {
conn, err := nats.Connect(testbus.URL(t))
if err != nil {
t.Fatal(err)
}
defer conn.Close()
sub, err := conn.Subscribe("pages.none", func(m *nats.Msg) { _ = m.Respond([]byte(`{"error":"gone"}`)) })
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe()
first, _ := json.Marshal(map[string]any{"error": "too large", "paged": Paged{Call: "call-1-1", Bytes: 10, Parts: 2}})
_, err = Whole(context.Background(), conn, "pages.none", first)
if err == nil || !strings.Contains(err.Error(), "part 0") || !strings.Contains(err.Error(), "gone") {
t.Fatalf("a missing part read as %v", err)
}
if plain, err := Whole(context.Background(), conn, "pages.none", []byte(`{"result":1}`)); err != nil ||
string(plain) != `{"result":1}` {
t.Fatalf("an answer that fits was changed: %s %v", plain, err)
}
}
internal/link/profile_report_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link_test
import (
internal/link/queue_integration_test.go
+220
View File
@@ -0,0 +1,220 @@
//go:build integration
package link
import (
"context"
"encoding/json"
"sync"
"testing"
"time"
"github.com/novox/mesh-controller/internal/broker"
)
func aBusWithACancelledSet(t *testing.T) *broker.JetStream {
t.Helper()
js := aBusWithTheBuildRole(t)
seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}}
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) })
return js
}
func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) {
js := aBusWithACancelledSet(t)
ctx, stop := context.WithCancel(context.Background())
defer stop()
if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil {
t.Fatal(err)
}
outcomes, err := js.Conn().SubscribeSync(BuildOutcome())
if err != nil {
t.Fatal(err)
}
defer func() { _ = outcomes.Unsubscribe() }()
_ = js.Conn().Flush()
for _, id := range []string{"build-cancelled", "build-kept"} {
body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"})
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
t.Fatal(err)
}
}
built := make(chan string, 2)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
built <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
select {
case id := <-built:
if id != "build-kept" {
t.Fatalf("%s was built", id)
}
case <-time.After(10 * time.Second):
t.Fatal("the ask that was not cancelled was never built")
}
msg, err := outcomes.NextMsg(5 * time.Second)
if err != nil {
t.Fatal(err)
}
var said BuildResult
if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" ||
said.Failed != CancelledByHand || said.On != "anchor" {
t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err)
}
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatal("the cancelled ask is still queued: terminated, it would not be")
}
func TestNatsAPausedHolderTakesNothingNew(t *testing.T) {
js := aBusWithACancelledSet(t)
ctx, stop := context.WithCancel(context.Background())
defer stop()
paused := make(chan bool, 1)
paused <- true
isPaused := func() bool {
p := <-paused
paused <- p
return p
}
took := make(chan string, 1)
machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused})
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"})
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
t.Fatalf("a paused holder took %s", id)
case <-time.After(3 * time.Second):
}
q, err := ReadQueue(ctx, js, TheBuildMachine)
if err != nil {
t.Fatal(err)
}
if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting {
t.Fatalf("while paused the queue reads %+v", q.Asks)
}
<-paused
paused <- false
select {
case id := <-took:
if id != "build-waits" {
t.Fatalf("took %s", id)
}
case <-time.After(10 * time.Second):
t.Fatal("resumed, the holder never took what waited")
}
}
func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) {
js := aBusWithTheBuildRole(t)
for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} {
body, _ := json.Marshal(s)
if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil {
t.Fatal(err)
}
}
said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"})
if err != nil {
t.Fatal(err)
}
if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 {
t.Fatalf("read back %+v", said)
}
}
func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) {
js := aBusWithACancelledSet(t)
was := cancelLook
cancelLook = 200 * time.Millisecond
t.Cleanup(func() { cancelLook = was })
ask := &natsBuilds{js: js, seat: TheBuildMachine}
go func() {
time.Sleep(500 * time.Millisecond)
_ = MarkCancelled(context.Background(), js, TheBuildMachine, "build-waited-cancelled")
}()
result, err := ask.Submit(context.Background(), BuildRequest{ID: "build-waited-cancelled", Repository: "/r"}, 10*time.Second)
if err != nil || result.Failed != CancelledByHand || result.ID != "build-waited-cancelled" {
t.Fatalf("the waiter heard %+v (%v)", result, err)
}
// Killed: the holder announces the failure as any outcome, and the waiter has it.
ctx, stop := context.WithCancel(context.Background())
defer stop()
machine := MachineOverNATS(js, "ace")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
r := work.Request()
_ = work.Announce(ctx, BuildResult{ID: r.ID, Repository: r.Repository, On: "ace", Failed: KilledByHand})
_ = work.Done()
})
}()
result, err = ask.Submit(context.Background(), BuildRequest{ID: "build-waited-killed", Repository: "/r"}, 10*time.Second)
if err != nil || result.Failed != KilledByHand || result.On != "ace" {
t.Fatalf("the waiter heard %+v (%v)", result, err)
}
}
func TestNatsAHolderPausedAsItsPullWasAnsweredHandsTheAskBack(t *testing.T) {
js := aBusWithACancelledSet(t)
ctx, stop := context.WithCancel(context.Background())
defer stop()
// Not paused when it asks; paused by the time the answer is read.
var looks int
var mu sync.Mutex
paused := func() bool {
mu.Lock()
defer mu.Unlock()
looks++
return looks > 1
}
took := make(chan string, 1)
machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: paused})
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID })
}()
time.Sleep(200 * time.Millisecond)
body, _ := json.Marshal(BuildRequest{ID: "build-handed-back", Repository: "/r"})
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
t.Fatalf("a holder paused as its pull was answered built %s", id)
case <-time.After(2 * time.Second):
}
q, err := ReadQueue(ctx, js, TheBuildMachine)
if err != nil {
t.Fatal(err)
}
if len(q.Asks) != 1 || q.Asks[0].ID != "build-handed-back" {
t.Fatalf("the ask is not back in the queue: %+v", q.Asks)
}
}
internal/link/queue_test.go
-207
View File
@@ -1,15 +1,10 @@
package link
import (
"context"
"encoding/json"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
)
// The build queue read and changed by hand (novox/hq ADR 0219).
@@ -121,216 +116,14 @@ func TestACancelledSetReadSaysWhatWasCancelled(t *testing.T) {
// --- against a real server ----------------------------------------------------------------------
func aBusWithACancelledSet(t *testing.T) *broker.JetStream {
t.Helper()
js := aBusWithTheBuildRole(t)
seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}}
if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) })
return js
}
// **The race a delete alone leaves open**: an ask fetched in the moment it was cancelled is ended by
// the holder that took it — terminated, never built, its outcome said as failed.
func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) {
js := aBusWithACancelledSet(t)
ctx, stop := context.WithCancel(context.Background())
defer stop()
if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil {
t.Fatal(err)
}
outcomes, err := js.Conn().SubscribeSync(BuildOutcome())
if err != nil {
t.Fatal(err)
}
defer func() { _ = outcomes.Unsubscribe() }()
_ = js.Conn().Flush()
for _, id := range []string{"build-cancelled", "build-kept"} {
body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"})
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
t.Fatal(err)
}
}
built := make(chan string, 2)
machine := MachineOverNATS(js, "anchor")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
built <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
select {
case id := <-built:
if id != "build-kept" {
t.Fatalf("%s was built", id)
}
case <-time.After(10 * time.Second):
t.Fatal("the ask that was not cancelled was never built")
}
msg, err := outcomes.NextMsg(5 * time.Second)
if err != nil {
t.Fatal(err)
}
var said BuildResult
if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" ||
said.Failed != CancelledByHand || said.On != "anchor" {
t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err)
}
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatal("the cancelled ask is still queued: terminated, it would not be")
}
// A paused holder takes nothing; resumed, it takes what waited.
func TestNatsAPausedHolderTakesNothingNew(t *testing.T) {
js := aBusWithACancelledSet(t)
ctx, stop := context.WithCancel(context.Background())
defer stop()
paused := make(chan bool, 1)
paused <- true
isPaused := func() bool {
p := <-paused
paused <- p
return p
}
took := make(chan string, 1)
machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused})
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
took <- work.Request().ID
_ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"})
_ = work.Done()
})
}()
body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"})
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
t.Fatalf("a paused holder took %s", id)
case <-time.After(3 * time.Second):
}
q, err := ReadQueue(ctx, js, TheBuildMachine)
if err != nil {
t.Fatal(err)
}
if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting {
t.Fatalf("while paused the queue reads %+v", q.Asks)
}
<-paused
paused <- false
select {
case id := <-took:
if id != "build-waits" {
t.Fatalf("took %s", id)
}
case <-time.After(10 * time.Second):
t.Fatal("resumed, the holder never took what waited")
}
}
// What a holder last said about taking work is read back per machine.
func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) {
js := aBusWithTheBuildRole(t)
for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} {
body, _ := json.Marshal(s)
if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil {
t.Fatal(err)
}
}
said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"})
if err != nil {
t.Fatal(err)
}
if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 {
t.Fatalf("read back %+v", said)
}
}
// A `build` waiting on its outcome hears a cancel — which publishes no outcome — from the cancelled
// set, and a kill from the outcome the holder announces (novox/hq ADR 0219).
func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) {
js := aBusWithACancelledSet(t)
was := cancelLook
cancelLook = 200 * time.Millisecond
t.Cleanup(func() { cancelLook = was })
ask := &natsBuilds{js: js, seat: TheBuildMachine}
go func() {
time.Sleep(500 * time.Millisecond)
_ = MarkCancelled(context.Background(), js, TheBuildMachine, "build-waited-cancelled")
}()
result, err := ask.Submit(context.Background(), BuildRequest{ID: "build-waited-cancelled", Repository: "/r"}, 10*time.Second)
if err != nil || result.Failed != CancelledByHand || result.ID != "build-waited-cancelled" {
t.Fatalf("the waiter heard %+v (%v)", result, err)
}
// Killed: the holder announces the failure as any outcome, and the waiter has it.
ctx, stop := context.WithCancel(context.Background())
defer stop()
machine := MachineOverNATS(js, "ace")
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) {
r := work.Request()
_ = work.Announce(ctx, BuildResult{ID: r.ID, Repository: r.Repository, On: "ace", Failed: KilledByHand})
_ = work.Done()
})
}()
result, err = ask.Submit(context.Background(), BuildRequest{ID: "build-waited-killed", Repository: "/r"}, 10*time.Second)
if err != nil || result.Failed != KilledByHand || result.On != "ace" {
t.Fatalf("the waiter heard %+v (%v)", result, err)
}
}
// Paused in the moment a pull was answered: the ask is handed back, not built (ADR 0219).
func TestNatsAHolderPausedAsItsPullWasAnsweredHandsTheAskBack(t *testing.T) {
js := aBusWithACancelledSet(t)
ctx, stop := context.WithCancel(context.Background())
defer stop()
// Not paused when it asks; paused by the time the answer is read.
var looks int
var mu sync.Mutex
paused := func() bool {
mu.Lock()
defer mu.Unlock()
looks++
return looks > 1
}
took := make(chan string, 1)
machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: paused})
defer machine.Close()
go func() {
_ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID })
}()
time.Sleep(200 * time.Millisecond)
body, _ := json.Marshal(BuildRequest{ID: "build-handed-back", Repository: "/r"})
if _, err := js.Context().Publish(BuildWork(), body); err != nil {
t.Fatal(err)
}
select {
case id := <-took:
t.Fatalf("a holder paused as its pull was answered built %s", id)
case <-time.After(2 * time.Second):
}
q, err := ReadQueue(ctx, js, TheBuildMachine)
if err != nil {
t.Fatal(err)
}
if len(q.Asks) != 1 || q.Asks[0].ID != "build-handed-back" {
t.Fatalf("the ask is not back in the queue: %+v", q.Asks)
}
}
internal/link/receive_nats_integration_test.go
+440
View File
@@ -0,0 +1,440 @@
//go:build integration
package link
import (
"context"
"encoding/json"
"errors"
"sync"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
func aBus(t *testing.T) *broker.JetStream {
t.Helper()
url := testbus.URL(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
// **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong.
//
// The streams are emptied rather than deleted and recreated, because delete-then-add is not a
// reset: the server's teardown races the creation, and a test then inherits the previous one's
// messages — which reads as a redelivery bug in the code under test.
//
// The consumers are removed, because deleting a stream used to take them with it and purging
// does not. A durable *push* consumer that survives between tests keeps pushing to a delivery
// subject the previous test's subscription has gone from: the messages count as delivered, go
// nowhere, and the next test waits out its timeout for an announcement the server believes it
// already sent. The controller recreates what it needs on start, so leaving none is correct.
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
clean := func() {
for _, c := range broker.MeshConsumers() {
_ = js.Context().DeleteConsumer(c.Stream, c.Name)
}
for _, s := range broker.MeshStreams() {
_ = js.Context().PurgeStream(s.Name)
}
}
clean()
t.Cleanup(clean)
return js
}
func TestNatsAReportIsHeardAndLeavesTheStream(t *testing.T) {
js := aBus(t)
store := &counted{}
_, stop := servingOn(t, js, store)
defer stop()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
eventually(t, "a report being recorded", func() bool { return store.count() == 1 })
eventually(t, "the report leaving the work queue", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
}
func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T) {
js := aBus(t)
store := &counted{}
store.refusing(errors.Join(ErrTryAgain, errors.New("the database system is starting up")))
_, stop := servingOn(t, js, store)
defer stop()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
// Held: the message is the server's, unacknowledged, and still in the stream.
eventually(t, "the report being redelivered at least once", func() bool {
info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
return err == nil && info.NumRedelivered >= 1
})
info, err := js.Context().StreamInfo("CONTROL")
if err != nil || info.State.Msgs != 1 {
t.Fatalf("a held report did not stay on the server: %+v, %v", info, err)
}
if store.count() != 0 {
t.Fatalf("a report was recorded by a store that was refusing it")
}
store.refusing(nil)
eventually(t, "the report being recorded once the store was back",
func() bool { return store.count() == 1 })
eventually(t, "the recorded report leaving the work queue", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
}
func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) {
js := aBus(t)
store := &sentAndHeardSafely{sent: "d2"}
_, stop := servingOn(t, js, store)
defer stop()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
eventually(t, "the superseded report leaving the stream", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
if store.count() != 0 {
t.Fatalf("a report about a superseded declaration was acted on")
}
}
func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) {
js := aBus(t)
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()},
enroller: enrolsAs{reply: EnrolReply{Accepted: true, Node: "anchor"}}, log: quiet()}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
inbox := nats.NewInbox()
answers, err := js.Conn().SubscribeSync(inbox)
if err != nil {
t.Fatal(err)
}
body, _ := json.Marshal(EnrolRequest{Node: "anchor", Secret: "t", ReplyTo: inbox})
if _, err := js.Context().Publish(EnrolSubject, body); err != nil {
t.Fatal(err)
}
msg, err := answers.NextMsg(8 * time.Second)
if err != nil {
t.Fatalf("no answer reached the address the request named: %v", err)
}
var reply EnrolReply
if err := json.Unmarshal(msg.Data, &reply); err != nil {
t.Fatal(err)
}
if !reply.Accepted || reply.Node != "anchor" {
t.Fatalf("the answer was not the mesh's: %+v", reply)
}
// And the address really is not the one the transport carried: what the consumer saw was its
// own ack subject, which is why this had to travel in the payload.
if msg.Subject != inbox {
t.Fatalf("the answer arrived on %s, not the address the request named", msg.Subject)
}
}
func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) {
js := aBus(t)
store := &counted{}
_, stop := servingOn(t, js, store)
defer stop()
// Given time to subscribe: a core subscription that is not yet up misses what is published,
// which is the guarantee a heartbeat has and not a fault.
eventually(t, "the heartbeat subscription coming up", func() bool {
body, _ := json.Marshal(Alive{Node: "anchor"})
_ = js.Conn().Publish(AliveSubject("anchor"), body)
_ = js.Conn().Flush()
return store.count() >= 1
})
info, err := js.Context().StreamInfo("CONTROL")
if err != nil || info.State.Msgs != 0 {
t.Fatalf("a heartbeat was persisted, and the mesh's least valuable message now competes "+
"for retention with its most valuable: %+v, %v", info, err)
}
}
func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T) {
js := aBus(t)
told := &toldAbout{}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(told); err != nil {
t.Fatal(err)
}
if err := s.Answers(replaysWith{}); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
// The controller's event consumer is made from now (novox/hq issue 248): what was published before
// it existed is history it never replays. So the announcements are made once it is there.
eventually(t, "the controller's event consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil
})
moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"})
if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(broker.ControllerFollows[1], []byte(`{}`)); err != nil {
t.Fatal(err)
}
eventually(t, "the catalogue's upgrade reaching the controller",
func() bool { return told.count() == 1 })
eventually(t, "both announcements being acknowledged", func() bool {
info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2
})
}
func TestNatsAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) {
js := aBus(t)
store := &counted{}
store.refusing(errors.Join(ErrTryAgain, errors.New("connection refused")))
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()},
listener: store, log: quiet(), giveUp: 1500 * time.Millisecond}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
eventually(t, "the report being let go once the bound passed", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
if store.count() != 0 {
t.Fatalf("a report was recorded by a store that never came back")
}
}
func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
// Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq
// issue 717: the test once set its window before the serving loop asserted the controller's own of thirty
// seconds, so it waited nine seconds and passed with the heartbeat removed).
before := heartbeatWhileWorking
heartbeatWhileWorking = 250 * time.Millisecond
t.Cleanup(func() { heartbeatWhileWorking = before })
js := aBus(t)
var mu sync.Mutex
handled := 0
slow := make(chan struct{})
s, stop := servingOn(t, js, nil)
defer stop()
s.listener = slowly{func() {
mu.Lock()
handled++
first := handled == 1
mu.Unlock()
if first {
time.Sleep(3 * time.Second)
close(slow)
}
}}
// The controller's own consumer, given a window short enough to outlive in a test once the serving loop has
// asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds,
// which is how this test once waited out a window it never had.
for deadline := time.Now().Add(10 * time.Second); ; {
info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
if err == nil && info.PushBound {
break
}
if time.Now().After(deadline) {
t.Fatalf("the serving loop never bound the controller's consumer: %v", err)
}
time.Sleep(10 * time.Millisecond)
}
if err := js.EnsureConsumer(broker.Consumer{
Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1,
Why: "a window short enough to outlive in a test",
}); err != nil {
t.Fatal(err)
}
if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second {
t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err)
}
body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
select {
case <-slow:
case <-time.After(30 * time.Second):
t.Fatal("the slow work never finished")
}
// A moment for a redelivery to arrive, if the bus were going to send one.
time.Sleep(1500 * time.Millisecond)
mu.Lock()
defer mu.Unlock()
if handled != 1 {
t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled)
}
// Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone
// would pass on a bus that redelivered the work every window; the bus's own count says it did not.
info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
if err != nil {
t.Fatal(err)
}
if info.NumRedelivered != 0 {
t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered)
}
}
func TestNatsTheMeshSaysWhatAMachineApplied(t *testing.T) {
js := aBus(t)
heard := make(chan *nats.Msg, 4)
sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) {
heard <- m
})
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection
_, stop := servingOn(t, js, &counted{})
defer stop()
// A report that changed something: the store says it was news.
body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store", "broker"}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
select {
case m := <-heard:
if m.Subject != SeatEventSubject(MeshControllerSeat, KeyApplied) {
t.Fatalf("the mesh stated %q", m.Subject)
}
var said Applied
if err := json.Unmarshal(m.Data, &said); err != nil {
t.Fatal(err)
}
if said.Node != "anchor" || said.Declared != "d1" || said.Resources != 2 {
t.Fatalf("it said %+v", said)
}
case <-time.After(10 * time.Second):
t.Fatal("the mesh said nothing about a machine that now runs something else")
}
// A refusal is its own fact, with the reason in it rather than only in a log.
refusal, err := json.Marshal(Report{Node: "anchor", Declared: "d2",
Failed: map[string]string{"gitea.server": "no such image"}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(ReportSubject("anchor"), refusal); err != nil {
t.Fatal(err)
}
select {
case m := <-heard:
if m.Subject != SeatEventSubject(MeshControllerSeat, KeyRefused) {
t.Fatalf("a refusal was stated as %q", m.Subject)
}
var said Refused
if err := json.Unmarshal(m.Data, &said); err != nil {
t.Fatal(err)
}
if said.Failed["gitea.server"] == "" {
t.Fatalf("the refusal does not say which resource or why: %+v", said)
}
case <-time.After(10 * time.Second):
t.Fatal("the mesh said nothing about a machine that refused what it was sent")
}
}
func TestNatsAReportThatIsNotNewsIsNotStated(t *testing.T) {
js := aBus(t)
heard := make(chan *nats.Msg, 4)
sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) {
heard <- m
})
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection
// A store that records the report and says it was nothing new — which is what the mesh's own
// store says about a machine repeating itself.
_, stop := servingOn(t, js, sameAgain{})
defer stop()
body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
select {
case m := <-heard:
t.Fatalf("the mesh stated %q about a machine that changed nothing", m.Subject)
case <-time.After(3 * time.Second):
}
}
func TestNatsAMergeHeldMoreThanFiveTimesIsStillHandedOverAndRecorded(t *testing.T) {
first, most := MergeHeldFirst, MergeHeldMost
MergeHeldFirst, MergeHeldMost = 5*time.Millisecond, 40*time.Millisecond
t.Cleanup(func() { MergeHeldFirst, MergeHeldMost = first, most })
js := aBus(t)
up := &manyTimesBusy{busy: 8}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(up); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
eventually(t, "the merges consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
return err == nil
})
body, _ := json.Marshal(SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"})
if _, err := js.Context().Publish(MergedSubject, body); err != nil {
t.Fatal(err)
}
eventually(t, "the ninth hand-over recording the merge", func() bool { return up.called() == 9 })
eventually(t, "the merge being acknowledged", func() bool {
info, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
return err == nil && info.NumAckPending == 0 && info.NumPending == 0
})
if info, err := js.Context().StreamInfo(broker.DeadLettersStream); err == nil && info.State.Msgs != 0 {
t.Fatalf("%d message(s) in dead letters: a merge was given up on", info.State.Msgs)
}
}
internal/link/receive_nats_test.go
-419
View File
@@ -2,7 +2,6 @@ package link
import (
"context"
"encoding/json"
"errors"
"io"
"log"
@@ -10,11 +9,7 @@ import (
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/testbus"
)
// The consume side against a real server, because what is being checked is what the server does.
@@ -26,42 +21,6 @@ import (
//
// go test ./internal/link/ -run TestNats (each test on a bus of its own: internal/testbus)
func aBus(t *testing.T) *broker.JetStream {
t.Helper()
url := testbus.URL(t)
js, err := broker.Dial(url)
if err != nil {
t.Fatal(err)
}
t.Cleanup(js.Close)
// **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong.
//
// The streams are emptied rather than deleted and recreated, because delete-then-add is not a
// reset: the server's teardown races the creation, and a test then inherits the previous one's
// messages — which reads as a redelivery bug in the code under test.
//
// The consumers are removed, because deleting a stream used to take them with it and purging
// does not. A durable *push* consumer that survives between tests keeps pushing to a delivery
// subject the previous test's subscription has gone from: the messages count as delivered, go
// nowhere, and the next test waits out its timeout for an announcement the server believes it
// already sent. The controller recreates what it needs on start, so leaving none is correct.
if err := broker.AssertMeshStreams(js); err != nil {
t.Fatal(err)
}
clean := func() {
for _, c := range broker.MeshConsumers() {
_ = js.Context().DeleteConsumer(c.Stream, c.Name)
}
for _, s := range broker.MeshStreams() {
_ = js.Context().PurgeStream(s.Name)
}
}
clean()
t.Cleanup(clean)
return js
}
// serving1 is a controller reading from a real bus, and a way to stop it.
func servingOn(t *testing.T, js *broker.JetStream, l Listener) (*Server, func()) {
t.Helper()
@@ -122,81 +81,13 @@ func eventually(t *testing.T, what string, is func() bool) {
// stream does not hold it. A work queue is the check: what is acknowledged leaves it.
func quiet() *log.Logger { return log.New(io.Discard, "", 0) }
func TestNatsAReportIsHeardAndLeavesTheStream(t *testing.T) {
js := aBus(t)
store := &counted{}
_, stop := servingOn(t, js, store)
defer stop()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
eventually(t, "a report being recorded", func() bool { return store.count() == 1 })
eventually(t, "the report leaving the work queue", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
}
// **The store window, in the server.** A report the store cannot take is naked with a delay and
// comes back; once the store is there it is recorded and leaves the stream. The controller holds
// nothing in the meantime — which is what the sequence check below is for: the message is still on
// the server while it waits.
func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T) {
js := aBus(t)
store := &counted{}
store.refusing(errors.Join(ErrTryAgain, errors.New("the database system is starting up")))
_, stop := servingOn(t, js, store)
defer stop()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
// Held: the message is the server's, unacknowledged, and still in the stream.
eventually(t, "the report being redelivered at least once", func() bool {
info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
return err == nil && info.NumRedelivered >= 1
})
info, err := js.Context().StreamInfo("CONTROL")
if err != nil || info.State.Msgs != 1 {
t.Fatalf("a held report did not stay on the server: %+v, %v", info, err)
}
if store.count() != 0 {
t.Fatalf("a report was recorded by a store that was refusing it")
}
store.refusing(nil)
eventually(t, "the report being recorded once the store was back",
func() bool { return store.count() == 1 })
eventually(t, "the recorded report leaving the work queue", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
}
// A report about a declaration the mesh has moved past is settled without being acted on, and
// leaves the stream rather than coming back for ever.
func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) {
js := aBus(t)
store := &sentAndHeardSafely{sent: "d2"}
_, stop := servingOn(t, js, store)
defer stop()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
eventually(t, "the superseded report leaving the stream", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
if store.count() != 0 {
t.Fatalf("a report about a superseded declaration was acted on")
}
}
// sentAndHeardSafely is sentAndHeard, read from two goroutines.
type sentAndHeardSafely struct {
@@ -228,41 +119,6 @@ func (s *sentAndHeardSafely) count() int {
// that the answer arrives there — which is the whole reason the address is a field rather than the
// transport's reply, and this is the test design 25 §2 asks for so the reason cannot quietly become
// folklore.
func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) {
js := aBus(t)
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()},
enroller: enrolsAs{reply: EnrolReply{Accepted: true, Node: "anchor"}}, log: quiet()}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
inbox := nats.NewInbox()
answers, err := js.Conn().SubscribeSync(inbox)
if err != nil {
t.Fatal(err)
}
body, _ := json.Marshal(EnrolRequest{Node: "anchor", Secret: "t", ReplyTo: inbox})
if _, err := js.Context().Publish(EnrolSubject, body); err != nil {
t.Fatal(err)
}
msg, err := answers.NextMsg(8 * time.Second)
if err != nil {
t.Fatalf("no answer reached the address the request named: %v", err)
}
var reply EnrolReply
if err := json.Unmarshal(msg.Data, &reply); err != nil {
t.Fatal(err)
}
if !reply.Accepted || reply.Node != "anchor" {
t.Fatalf("the answer was not the mesh's: %+v", reply)
}
// And the address really is not the one the transport carried: what the consumer saw was its
// own ack subject, which is why this had to travel in the payload.
if msg.Subject != inbox {
t.Fatalf("the answer arrived on %s, not the address the request named", msg.Subject)
}
}
// enrolsAs answers every request the same way.
type enrolsAs struct{ reply EnrolReply }
@@ -271,26 +127,6 @@ func (e enrolsAs) Enrol(context.Context, EnrolRequest) (EnrolReply, error) { ret
// A heartbeat is core NATS: it reaches the controller and nothing is persisted, so the stream the
// reports live in stays empty.
func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) {
js := aBus(t)
store := &counted{}
_, stop := servingOn(t, js, store)
defer stop()
// Given time to subscribe: a core subscription that is not yet up misses what is published,
// which is the guarantee a heartbeat has and not a fault.
eventually(t, "the heartbeat subscription coming up", func() bool {
body, _ := json.Marshal(Alive{Node: "anchor"})
_ = js.Conn().Publish(AliveSubject("anchor"), body)
_ = js.Conn().Flush()
return store.count() >= 1
})
info, err := js.Context().StreamInfo("CONTROL")
if err != nil || info.State.Msgs != 0 {
t.Fatalf("a heartbeat was persisted, and the mesh's least valuable message now competes "+
"for retention with its most valuable: %+v, %v", info, err)
}
}
// The two events the controller follows arrive over one durable consumer with two filters, and it
// can acknowledge them.
@@ -299,41 +135,6 @@ func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) {
// is the first thing in the mesh to use one; and a delivery from the events stream is acknowledged
// on a different ack subject from a delivery from the control stream, which the controller's own
// permission list has to cover or every announcement is redelivered for ever.
func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T) {
js := aBus(t)
told := &toldAbout{}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(told); err != nil {
t.Fatal(err)
}
if err := s.Answers(replaysWith{}); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
// The controller's event consumer is made from now (novox/hq issue 248): what was published before
// it existed is history it never replays. So the announcements are made once it is there.
eventually(t, "the controller's event consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil
})
moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"})
if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(broker.ControllerFollows[1], []byte(`{}`)); err != nil {
t.Fatal(err)
}
eventually(t, "the catalogue's upgrade reaching the controller",
func() bool { return told.count() == 1 })
eventually(t, "both announcements being acknowledged", func() bool {
info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2
})
}
type toldAbout struct {
mu sync.Mutex
@@ -360,28 +161,6 @@ func (u *toldAbout) count() int {
// A store that never comes back: the report is let go once the bound passes, and it leaves the
// stream rather than being held for ever. The bound is the controller's, not the server's — nothing
// here sets max-deliver, and that is deliberate (streams.go).
func TestNatsAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) {
js := aBus(t)
store := &counted{}
store.refusing(errors.Join(ErrTryAgain, errors.New("connection refused")))
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()},
listener: store, log: quiet(), giveUp: 1500 * time.Millisecond}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
eventually(t, "the report being let go once the bound passed", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
if store.count() != 0 {
t.Fatalf("a report was recorded by a store that never came back")
}
}
// A merge announcement is not what these tests are about; taken and forgotten.
func (t *toldAbout) SourceMoved(context.Context, SourceMoved) error { return nil }
@@ -406,81 +185,6 @@ func TestEverySubjectTheControllerFollowsDecodesToAKind(t *testing.T) {
// a consumer that died and wrong for one that is busy. Acting on a merge builds modules — minutes
// against a thirty-second window — and the same merge was handed over five times while the first
// build was still running (2026-09-28). Here the window is a second and the work takes three.
func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) {
// Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq
// issue 717: the test once set its window before the serving loop asserted the controller's own of thirty
// seconds, so it waited nine seconds and passed with the heartbeat removed).
before := heartbeatWhileWorking
heartbeatWhileWorking = 250 * time.Millisecond
t.Cleanup(func() { heartbeatWhileWorking = before })
js := aBus(t)
var mu sync.Mutex
handled := 0
slow := make(chan struct{})
s, stop := servingOn(t, js, nil)
defer stop()
s.listener = slowly{func() {
mu.Lock()
handled++
first := handled == 1
mu.Unlock()
if first {
time.Sleep(3 * time.Second)
close(slow)
}
}}
// The controller's own consumer, given a window short enough to outlive in a test once the serving loop has
// asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds,
// which is how this test once waited out a window it never had.
for deadline := time.Now().Add(10 * time.Second); ; {
info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
if err == nil && info.PushBound {
break
}
if time.Now().After(deadline) {
t.Fatalf("the serving loop never bound the controller's consumer: %v", err)
}
time.Sleep(10 * time.Millisecond)
}
if err := js.EnsureConsumer(broker.Consumer{
Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1,
Why: "a window short enough to outlive in a test",
}); err != nil {
t.Fatal(err)
}
if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second {
t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err)
}
body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
select {
case <-slow:
case <-time.After(30 * time.Second):
t.Fatal("the slow work never finished")
}
// A moment for a redelivery to arrive, if the bus were going to send one.
time.Sleep(1500 * time.Millisecond)
mu.Lock()
defer mu.Unlock()
if handled != 1 {
t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled)
}
// Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone
// would pass on a bus that redelivered the work every window; the bus's own count says it did not.
info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName)
if err != nil {
t.Fatal(err)
}
if info.NumRedelivered != 0 {
t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered)
}
}
// slowly is a listener that runs whatever it was given.
type slowly struct{ work func() }
@@ -490,101 +194,9 @@ func (s slowly) Heard(context.Context, Report) (bool, error) { s.work(); return
// **The mesh says what it applied** (novox/hq ADR 0134), under the seat the control plane holds — and
// says nothing when a report is the same state said again, which is what a machine reconciling every
// minute sends.
func TestNatsTheMeshSaysWhatAMachineApplied(t *testing.T) {
js := aBus(t)
heard := make(chan *nats.Msg, 4)
sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) {
heard <- m
})
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection
_, stop := servingOn(t, js, &counted{})
defer stop()
// A report that changed something: the store says it was news.
body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store", "broker"}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
select {
case m := <-heard:
if m.Subject != SeatEventSubject(MeshControllerSeat, KeyApplied) {
t.Fatalf("the mesh stated %q", m.Subject)
}
var said Applied
if err := json.Unmarshal(m.Data, &said); err != nil {
t.Fatal(err)
}
if said.Node != "anchor" || said.Declared != "d1" || said.Resources != 2 {
t.Fatalf("it said %+v", said)
}
case <-time.After(10 * time.Second):
t.Fatal("the mesh said nothing about a machine that now runs something else")
}
// A refusal is its own fact, with the reason in it rather than only in a log.
refusal, err := json.Marshal(Report{Node: "anchor", Declared: "d2",
Failed: map[string]string{"gitea.server": "no such image"}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(ReportSubject("anchor"), refusal); err != nil {
t.Fatal(err)
}
select {
case m := <-heard:
if m.Subject != SeatEventSubject(MeshControllerSeat, KeyRefused) {
t.Fatalf("a refusal was stated as %q", m.Subject)
}
var said Refused
if err := json.Unmarshal(m.Data, &said); err != nil {
t.Fatal(err)
}
if said.Failed["gitea.server"] == "" {
t.Fatalf("the refusal does not say which resource or why: %+v", said)
}
case <-time.After(10 * time.Second):
t.Fatal("the mesh said nothing about a machine that refused what it was sent")
}
}
// And a report that is not news is not a fact. A machine reconciles every minute; a fact per report
// would be a fact per minute per machine, which is a stream nobody reads.
func TestNatsAReportThatIsNotNewsIsNotStated(t *testing.T) {
js := aBus(t)
heard := make(chan *nats.Msg, 4)
sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) {
heard <- m
})
if err != nil {
t.Fatal(err)
}
defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection
// A store that records the report and says it was nothing new — which is what the mesh's own
// store says about a machine repeating itself.
_, stop := servingOn(t, js, sameAgain{})
defer stop()
body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
select {
case m := <-heard:
t.Fatalf("the mesh stated %q about a machine that changed nothing", m.Subject)
case <-time.After(3 * time.Second):
}
}
// sameAgain records a report and says it was the same state said again.
type sameAgain struct{}
@@ -617,37 +229,6 @@ func (u *manyTimesBusy) called() int {
// **A merge held more than five times is still handed over and recorded** (issue 973, novox/hq ADR 0407 decision
// 3): the merges have a durable consumer of their own with no max-deliver, so they never go to DEAD_LETTERS to wait
// for a person; the delay grows, and is capped.
func TestNatsAMergeHeldMoreThanFiveTimesIsStillHandedOverAndRecorded(t *testing.T) {
first, most := MergeHeldFirst, MergeHeldMost
MergeHeldFirst, MergeHeldMost = 5*time.Millisecond, 40*time.Millisecond
t.Cleanup(func() { MergeHeldFirst, MergeHeldMost = first, most })
js := aBus(t)
up := &manyTimesBusy{busy: 8}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(up); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
eventually(t, "the merges consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
return err == nil
})
body, _ := json.Marshal(SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"})
if _, err := js.Context().Publish(MergedSubject, body); err != nil {
t.Fatal(err)
}
eventually(t, "the ninth hand-over recording the merge", func() bool { return up.called() == 9 })
eventually(t, "the merge being acknowledged", func() bool {
info, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
return err == nil && info.NumAckPending == 0 && info.NumPending == 0
})
if info, err := js.Context().StreamInfo(broker.DeadLettersStream); err == nil && info.State.Msgs != 0 {
t.Fatalf("%d message(s) in dead letters: a merge was given up on", info.State.Msgs)
}
}
// The delay a held merge waits grows from the first to the most and stays there.
func TestAHeldMergesDelayGrowsAndIsCapped(t *testing.T) {
internal/link/rekey_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link_test
import (
internal/link/replay314_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/replay330_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/seat_as_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/setuid_search_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/standby_nats_test.go
+2
View File
@@ -1,3 +1,5 @@
//go:build integration
package link
import (
internal/link/standing_integration_test.go
+48
View File
@@ -0,0 +1,48 @@
//go:build integration
package link
import (
"context"
"testing"
"github.com/novox/mesh-controller/internal/broker"
)
func TestNatsAProvidersStandingReachesTheController(t *testing.T) {
js := aBus(t)
kept := &lockedStandings{}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(&toldAbout{}); err != nil {
t.Fatal(err)
}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
eventually(t, "the controller's event consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil
})
for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} {
if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event,
[]byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil {
t.Fatal(err)
}
}
// Somebody else's event under the same prefix is not the controller's to hear.
if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil {
t.Fatal(err)
}
eventually(t, "both standings being kept, in order, naming the emitter", func() bool {
got := kept.all()
return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing
})
eventually(t, "both being acknowledged and nothing else delivered", func() bool {
info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2
})
}
internal/link/standing_test.go
-37
View File
@@ -150,43 +150,6 @@ func TestAStandingWithNothingKeepingItIsTaken(t *testing.T) {
// Over a real bus: a provider's standing published under its own module's namespace reaches the
// controller through the events consumer's filter — the one wildcard filter on it — names the emitter
// from the subject, and is acknowledged.
func TestNatsAProvidersStandingReachesTheController(t *testing.T) {
js := aBus(t)
kept := &lockedStandings{}
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()}
if err := s.Follows(&toldAbout{}); err != nil {
t.Fatal(err)
}
if err := s.Watches(kept); err != nil {
t.Fatal(err)
}
ctx, stop := context.WithCancel(context.Background())
defer stop()
go func() { _ = s.Serve(ctx) }()
eventually(t, "the controller's event consumer being made", func() bool {
_, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil
})
for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} {
if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event,
[]byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil {
t.Fatal(err)
}
}
// Somebody else's event under the same prefix is not the controller's to hear.
if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil {
t.Fatal(err)
}
eventually(t, "both standings being kept, in order, naming the emitter", func() bool {
got := kept.all()
return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing
})
eventually(t, "both being acknowledged and nothing else delivered", func() bool {
info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName)
return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2
})
}
type lockedStandings struct {
mu sync.Mutex