Compare commits
118
Commits
54070f5bc3
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
caa3fc8995 | ||
|
|
a0c971cea9 | ||
|
|
8802076499 | ||
|
|
2d2e736b45 | ||
|
|
fdfda0e839 | ||
|
|
540cda11df | ||
|
|
ad753c70f6 | ||
|
|
da20fea549 | ||
|
|
2f4f50b9ca | ||
|
|
14f405ece5 | ||
|
|
0d1d780d43 | ||
|
|
5c807808e8 | ||
|
|
da5a5c9b67 | ||
|
|
07a8f04e65 | ||
|
|
dd802c81a7 | ||
|
|
a5a43c8f31 | ||
|
|
3e7f34d041 | ||
|
|
e29e855bdf | ||
|
|
da3c01089e | ||
|
|
253283d68e | ||
|
|
3f038b989e | ||
|
|
3834c72747 | ||
|
|
68088ec782 | ||
|
|
8da2619920 | ||
|
|
59c8cf2f24 | ||
|
|
3ac8657a49 | ||
|
|
230f0a1a18 | ||
|
|
6e487fa01f | ||
|
|
7a74cee83f | ||
|
|
378e65780d | ||
|
|
296e292f55 | ||
|
|
791b92c875 | ||
|
|
1490c3b075 | ||
|
|
4104d49ec8 | ||
|
|
1991fe931b | ||
|
|
d8249dad8b | ||
|
|
df4d1204c7 | ||
|
|
e8c8f0488b | ||
|
|
cfb6a8c531 | ||
|
|
9b7f9a7fd2 | ||
|
|
65632e7999 | ||
|
|
f6a48ac5ef | ||
|
|
93a82cb032 | ||
|
|
de651e0fd1 | ||
|
|
97ee33e8f4 | ||
|
|
19e3eac746 | ||
|
|
863db82737 | ||
|
|
a069898c0c | ||
|
|
ffad53aad2 | ||
|
|
89cfa9431d | ||
|
|
5939bbdefe | ||
|
|
ec85cdf70c | ||
|
|
cd303f68ed | ||
|
|
2cbebbc011 | ||
|
|
ac79e29319 | ||
|
|
ef0ca69cb0 | ||
|
|
cec26d8596 | ||
|
|
e621e15d2f | ||
|
|
a08498b240 | ||
|
|
39ae7de6d3 | ||
|
|
4e12fc19b7 | ||
|
|
19f15d8ea0 | ||
|
|
b1cfb85f86 | ||
|
|
75a27ec1a4 | ||
|
|
85af245f1a | ||
|
|
1ec27e6ba1 | ||
|
|
425134784a | ||
|
|
fb66608a13 | ||
|
|
7be2df627f | ||
|
|
cb006fc8dd | ||
|
|
0273609090 | ||
|
|
3b49d7044f | ||
|
|
ba59283b4b | ||
|
|
461fa9c7b4 | ||
|
|
06bf8cb2fb | ||
|
|
222c9c1d72 | ||
|
|
0520773412 | ||
|
|
eba864dc5f | ||
|
|
2d3d5ab616 | ||
|
|
dc65fe2319 | ||
|
|
fc43238267 | ||
|
|
8c8bc5021e | ||
|
|
557c11af1e | ||
|
|
661e8eff2f | ||
|
|
8c0fa40e65 | ||
|
|
6746ab6492 | ||
|
|
48e9656c67 | ||
|
|
a825a4c904 | ||
|
|
315c622386 | ||
|
|
d3556f452e | ||
|
|
68db18bdfc | ||
|
|
cbc6bac8c0 | ||
|
|
0b4cd46b3f | ||
|
|
7f074d9c94 | ||
|
|
e76ba966b3 | ||
|
|
da6d038e07 | ||
|
|
c795915315 | ||
|
|
aa02e7bf4e | ||
|
|
4eeba8fb20 | ||
|
|
d095a3dcef | ||
|
|
5b08eb09be | ||
|
|
472744bbaa | ||
|
|
9859923664 | ||
|
|
fdab58ae4f | ||
|
|
3b989d8bb3 | ||
|
|
88ea361bf6 | ||
|
|
0149a3d7de | ||
|
|
b56c1d00a3 | ||
|
|
e16cb4207b | ||
|
|
a47c63799e | ||
|
|
703badcf5a | ||
|
|
cb00bbbfcf | ||
|
|
0e4feb0d3a | ||
|
|
275ddd76e8 | ||
|
|
e0fb241263 | ||
|
|
f89d870308 | ||
|
|
36c4ce605e | ||
|
|
fc55785956 |
@@ -432,6 +432,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
||||
}
|
||||
result.Against = built.Against
|
||||
result.SourceFingerprint = built.Source
|
||||
result.SourceInputs = built.SourceInputs
|
||||
// What the walk did instead of building, or why it built (novox/hq ADR 0350 §3, §4).
|
||||
result.ReuseKey = built.ReuseKey
|
||||
if built.Reused != nil {
|
||||
|
||||
@@ -33,7 +33,7 @@ const packageManagerKey = "the package manager ("
|
||||
func watchApplyFailing(f *signalFacts) []conditions.Observation {
|
||||
heard := map[string]bool{}
|
||||
for _, m := range f.machines {
|
||||
if !m.asleep() && !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) {
|
||||
if !m.asleep() && !m.lastHeard.IsZero() && !m.silent(f.now) {
|
||||
heard[m.name] = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -201,7 +201,7 @@ func routerRecordOf(conn *nats.Conn, inv *inventory.Inventory) func(ctx context.
|
||||
}
|
||||
|
||||
// asksRecords is the bucket the asks seat's declarer keeps its record of asks in.
|
||||
func asksRecords(ctx context.Context, inv *inventory.Inventory) (string, error) {
|
||||
func asksRecords(ctx context.Context, inv catalogueReader) (string, error) {
|
||||
declared, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/overlay"
|
||||
)
|
||||
|
||||
@@ -146,8 +147,18 @@ func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.
|
||||
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
register(t, open, aKeeper())
|
||||
pushedBy(t, open, "laptop")
|
||||
// What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only
|
||||
// when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown.
|
||||
if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, "carried", map[string]string{overlay.Name: "c-overlay"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -46,7 +46,21 @@ func pushedBy(t *testing.T, open *stores, node string) string {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := digestOf(body)
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, declared.Builds); err != nil {
|
||||
// A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was
|
||||
// built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109).
|
||||
builds := declared.Builds
|
||||
if builds == nil {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
names := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
builds = carriedBuilds(names, nil, current, nil)
|
||||
}
|
||||
if err := open.inventory.RecordSent(ctx, record.ID, digest, builds); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return digest
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send
|
||||
// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent
|
||||
// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so.
|
||||
|
||||
// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine
|
||||
// is away once presence says so). A machine
|
||||
// never heard from has not joined and is not away.
|
||||
func awayOf(machines []machineFacts, now time.Time) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
if m.silent(now) {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and
|
||||
// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did
|
||||
// before: a machine is not passed on a reading that was not made.
|
||||
var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer js.Close()
|
||||
machines := make([]machineFacts, len(nodes))
|
||||
for i, n := range nodes {
|
||||
machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen}
|
||||
}
|
||||
if err := withPresence(ctx, js.Conn(), machines, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return awayOf(machines, now), nil
|
||||
}
|
||||
|
||||
// rememberAway keeps on the plan the machines of running that are away: sorted, each once.
|
||||
func rememberAway(p *inventory.Plan, running []string, away map[string]bool) {
|
||||
for _, n := range running {
|
||||
if away[n] && !slices.Contains(p.Away, n) {
|
||||
p.Away = append(p.Away, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(p.Away)
|
||||
}
|
||||
|
||||
// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and
|
||||
// those it did not pass as away.
|
||||
func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string {
|
||||
return slices.DeleteFunc(slices.Clone(running), func(n string) bool {
|
||||
if !slices.Contains(p.Away, n) {
|
||||
return false
|
||||
}
|
||||
_, sent := s.Rest[n]
|
||||
return !sent
|
||||
})
|
||||
}
|
||||
|
||||
// awaySaying is what a plan says of the machines it passed as away, empty when it passed none.
|
||||
func awaySaying(p inventory.Plan) string {
|
||||
var said []string
|
||||
for _, n := range p.Away {
|
||||
said = append(said, n+" was away: sent when it is heard again")
|
||||
}
|
||||
return strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot.
|
||||
func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool {
|
||||
away, err := judgeAway(ctx, open, now)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err)
|
||||
return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above
|
||||
}
|
||||
return away
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1161/AC5, with the store and a real bus and no ticker: a plan of two tiers whose first tier runs on a machine away when it
|
||||
// is sent finishes that tier without waiting for it and without its bound, sends the second tier to the machine heard
|
||||
// alone, and says the machine was away (issue 1161). The machine away keeps its want: it was sent nothing of the plan's
|
||||
// builds, so the send it gets when it is heard again carries them (AC3).
|
||||
func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
// laptop is away as every tier is sent; anchor is heard.
|
||||
wasAway := judgeAway
|
||||
judgeAway = func(context.Context, *stores, time.Time) (map[string]bool, error) {
|
||||
return map[string]bool{"laptop": true}, nil
|
||||
}
|
||||
t.Cleanup(func() { judgeAway = wasAway })
|
||||
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = nil })
|
||||
sentHeard := make(chan *nats.Msg, 32)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
wasSaved := inventory.PlanSaved
|
||||
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||
|
||||
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
||||
var mu sync.Mutex
|
||||
var asks []string
|
||||
var sends [][]string
|
||||
wasAsk := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond))
|
||||
asks = append(asks, source.Repository+"#"+id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = wasAsk })
|
||||
digests := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
sends = append(sends, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builds := map[string]string{}
|
||||
for _, m := range []string{"a", "b"} {
|
||||
builds[m] = current[m].Commit
|
||||
}
|
||||
for _, node := range names {
|
||||
digests++
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
register := func(m, commit string, asked time.Time) {
|
||||
t.Helper()
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record := func(id, m, commit string, at time.Time) {
|
||||
t.Helper()
|
||||
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
for _, m := range []string{"a", "b"} {
|
||||
record("build-"+m+"-1", m, "c1", old)
|
||||
register(m, "c1", old)
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
||||
register("a", "c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// A machine's report on what it was sent, heard as the controller hears it.
|
||||
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||
seq := int64(0)
|
||||
reports := func(node string) {
|
||||
t.Helper()
|
||||
last, found, err := inv.SentTo(ctx, node)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s was sent nothing: %v", node, err)
|
||||
}
|
||||
seq++
|
||||
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
||||
"report_sequence": seq})
|
||||
var report link.Report
|
||||
if err := json.Unmarshal(body, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := listener.Heard(ctx, report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
plan := func() inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := inv.PlanByID(ctx, "plan-1160")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
built := time.Now().UTC()
|
||||
p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
||||
"b": {},
|
||||
}}
|
||||
first, stopFirst := context.WithCancel(ctx)
|
||||
defer stopFirst()
|
||||
startRolloutEvents(first, open, js)
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "tier 0 sent to anchor alone", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 1 && strings.Join(sends[0], ",") == "anchor"
|
||||
})
|
||||
reports("anchor")
|
||||
// The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own.
|
||||
var moved string
|
||||
eventually(t, "tier 1 reached on anchor's report alone", func() bool {
|
||||
if q := plan(); q.Tier == 1 {
|
||||
moved = q.Note
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") {
|
||||
t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved)
|
||||
}
|
||||
// The ask for tier 1's build is the next step, and records its build id on the module.
|
||||
eventually(t, "tier 1 asked", func() bool { m := plan().Modules["b"]; return m != nil && m.Build != "" })
|
||||
b := plan().Modules["b"]
|
||||
askedAt, _ := link.BuildAskedAt(b.Build)
|
||||
record(b.Build, "b", "c2", askedAt)
|
||||
register("b", "c2", askedAt)
|
||||
planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build)
|
||||
eventually(t, "tier 1 sent to anchor alone", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor"
|
||||
})
|
||||
// 1161/AC3: laptop was sent nothing of the plan's builds, so it still wants them for the send it gets when heard again.
|
||||
was, known, err := inv.SentBuilds(ctx, "laptop")
|
||||
if err != nil || !known {
|
||||
t.Fatalf("1161/AC3: laptop's last send: %v, known %v", err, known)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if was["a"] == current["a"].Commit || was["a"] != "c1" {
|
||||
t.Errorf("1161/AC3: laptop was last sent a at %q and a is now %q; want it still at c1, owed c2", was["a"], current["a"].Commit)
|
||||
}
|
||||
if got := plan().Away; strings.Join(got, ",") != "laptop" {
|
||||
t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so.
|
||||
|
||||
// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away.
|
||||
func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
machines := []machineFacts{
|
||||
{name: "heard", lastHeard: now.Add(-time.Minute)},
|
||||
{name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)},
|
||||
{name: "stuck", lastHeard: now, presence: presenceSilent},
|
||||
{name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard},
|
||||
{name: "new"},
|
||||
}
|
||||
got := awayOf(machines, now)
|
||||
for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false,
|
||||
"new": false} {
|
||||
if got[node] != want {
|
||||
t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once.
|
||||
func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) {
|
||||
p := &inventory.Plan{}
|
||||
away := map[string]bool{"g14": true, "ace": true}
|
||||
rememberAway(p, []string{"anchor", "g14"}, away)
|
||||
rememberAway(p, []string{"g14", "ace"}, away)
|
||||
if !slices.Equal(p.Away, []string{"ace", "g14"}) {
|
||||
t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC2: the plan says it, where its other passed machines are said.
|
||||
func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) {
|
||||
say := awaySaying(inventory.Plan{Away: []string{"g14"}})
|
||||
if !strings.Contains(say, "g14 was away: sent when it is heard again") {
|
||||
t.Fatalf("1161/AC2: the plan says %q", say)
|
||||
}
|
||||
if got := awaySaying(inventory.Plan{}); got != "" {
|
||||
t.Fatalf("1161/AC2: a plan that passed no machine says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for.
|
||||
func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) {
|
||||
p := inventory.Plan{Away: []string{"g14"}}
|
||||
none := &inventory.PlanModule{}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) {
|
||||
t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got)
|
||||
}
|
||||
sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) {
|
||||
t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -176,6 +176,11 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
repository, m.Base, m.Commit, already)
|
||||
return nil
|
||||
}
|
||||
if already == 0 {
|
||||
if done, err := f.firstBuilds(ctx, m, entries, read, now); err != nil || done {
|
||||
return err
|
||||
}
|
||||
}
|
||||
fmt.Print(nothingReadsIt(repository, m))
|
||||
return nil
|
||||
}
|
||||
@@ -219,10 +224,10 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
}
|
||||
if p.Batch() {
|
||||
if err := keepBatch(ctx, inv, &p, now, ""); err != nil {
|
||||
return notNow(err)
|
||||
return unsaved(err)
|
||||
}
|
||||
} else if err := nameMerges(ctx, inv, &p); err != nil {
|
||||
return notNow(err)
|
||||
return unsaved(err)
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s) after a later merge of its branch was put in %s (%s): that %s "+
|
||||
"answers it\n", repository, m.Base, m.Commit, p.ID, p.State, walkOrBatch(p))
|
||||
@@ -239,7 +244,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T
|
||||
}
|
||||
holdFor(&batch, m.Head, waits, holding, own, now)
|
||||
if err := keepBatch(ctx, inv, &batch, now, ""); err != nil {
|
||||
return notNow(err)
|
||||
return unsaved(err)
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID,
|
||||
batchWords(batch, now))
|
||||
@@ -362,7 +367,7 @@ func buildsEvery(p inventory.Plan, modules []inventory.Entry) bool {
|
||||
// ownPath says a merge touches a module on the controller's own path, whose walk waits for nobody's word.
|
||||
func ownPath(touches []inventory.Entry) bool {
|
||||
for _, e := range touches {
|
||||
if _, own := onTheControllersPath[e.Manifest.Module]; own {
|
||||
if onTheOwnPath(e.Manifest.Module, e.Manifest) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -742,6 +747,9 @@ func carriedOf(merges []inventory.BatchedMerge) ([]inventory.PlanCommit, []inven
|
||||
type keptMerge struct {
|
||||
link.SourceMoved
|
||||
Moves []string `json:"moves,omitempty"`
|
||||
// FirstBuild says the merge is a first merge into a repository the mesh builds nothing from, kept to ask the first
|
||||
// build of its modules once mesh-delivery's word on it is kept (novox/hq ADR 0432, issue 1146).
|
||||
FirstBuild bool `json:"first_build,omitempty"`
|
||||
}
|
||||
|
||||
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
||||
@@ -820,6 +828,12 @@ func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan
|
||||
// walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits.
|
||||
func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error {
|
||||
inv := open.inventory
|
||||
// A merge found in two records is the half-done move of a crash: put right before anything is decided from it.
|
||||
if repaired, err := inv.RepairMerges(ctx); err != nil {
|
||||
return err
|
||||
} else if repaired > 0 {
|
||||
fmt.Printf("%d merge(s) found in two plans, the half of a move a restart interrupted, are kept by the plan they were moved to\n", repaired)
|
||||
}
|
||||
if err := searchAfterFailures(ctx, inv); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -931,7 +945,11 @@ func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting [
|
||||
var read map[string][]inventory.ReadRepository
|
||||
var edges []inventory.Edge
|
||||
judged := false
|
||||
openWalks, err := openWalksOf(ctx, inv, started)
|
||||
onPath, err := ownPathNow(ctx, inv)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
openWalks, err := openWalksOf(ctx, inv, started, onPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -964,7 +982,7 @@ func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting [
|
||||
var fold, keep []inventory.Plan
|
||||
if !b.OwnPath() {
|
||||
for _, w := range waiting {
|
||||
ww, err := openWalkOf(ctx, inv, w)
|
||||
ww, err := openWalkOf(ctx, inv, w, onPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -999,7 +1017,7 @@ func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting [
|
||||
if !cut.Open() {
|
||||
continue
|
||||
}
|
||||
w, err := openWalkOf(ctx, inv, cut)
|
||||
w, err := openWalkOf(ctx, inv, cut, onPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1377,8 +1395,9 @@ func classOfModules(modules []string, entries []inventory.Entry) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
onPath := ownPathOf(entries)
|
||||
for _, m := range modules {
|
||||
if _, own := onTheControllersPath[m]; own || resolvers[m] {
|
||||
if _, own := onPath[m]; own || resolvers[m] {
|
||||
return inventory.ClassCore
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"io"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -152,6 +153,9 @@ func buildFrom(result link.BuildResult) inventory.Build {
|
||||
Path: result.Path,
|
||||
// What it was made from (novox/hq issue 280): two builds with one are one build.
|
||||
SourceFingerprint: result.SourceFingerprint,
|
||||
SourceInputs: result.SourceInputs,
|
||||
// And its reuse key (novox/hq ADR 0350 §1), which the chain names (ADR 0408 §1).
|
||||
ReuseKey: result.ReuseKey,
|
||||
}
|
||||
// When it was asked, which is what orders it against another build of the same module
|
||||
// (novox/hq 04-ISSUES/219) — not when it was heard.
|
||||
@@ -234,35 +238,56 @@ func buildsCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
|
||||
for _, b := range builds {
|
||||
what := b.Module
|
||||
if what == "" {
|
||||
// It failed before knowing what it was building, which is most of the interesting
|
||||
// failures. The repository is what a person has to go and look at.
|
||||
what = "?"
|
||||
}
|
||||
outcome := "built " + short(b.Commit)
|
||||
if !b.Worked() {
|
||||
outcome = "failed"
|
||||
}
|
||||
fmt.Printf("%-18s %-14s %-10s %s %s\n",
|
||||
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
||||
fmt.Printf(" %s", b.Repository)
|
||||
if b.Ref != "" {
|
||||
fmt.Printf(" at %s", b.Ref)
|
||||
}
|
||||
fmt.Println()
|
||||
for _, made := range b.Made {
|
||||
fmt.Printf(" %-10s %s\n", made.Kind, made.Reference)
|
||||
}
|
||||
if !b.Worked() {
|
||||
// The builder's own first line. The whole failure is often a build log, and printing
|
||||
// it here would bury every other row.
|
||||
fmt.Printf(" %s\n", firstLine(b.Failed))
|
||||
}
|
||||
printBuild(os.Stdout, b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// printBuild says one build: what, its outcome, where and when, what it made and published, its reuse key, and its
|
||||
// source fingerprint (novox/hq issue 1204: the one tool that reads what the builder recorded it was made from).
|
||||
func printBuild(w io.Writer, b inventory.Build) {
|
||||
what := b.Module
|
||||
if what == "" {
|
||||
// It failed before knowing what it was building, which is most of the interesting
|
||||
// failures. The repository is what a person has to go and look at.
|
||||
what = "?"
|
||||
}
|
||||
outcome := "built " + short(b.Commit)
|
||||
if !b.Worked() {
|
||||
outcome = "failed"
|
||||
}
|
||||
fmt.Fprintf(w, "%-18s %-14s %-10s %s %s\n",
|
||||
what, outcome, b.On, b.At.Local().Format("2006-01-02 15:04"), b.ID)
|
||||
fmt.Fprintf(w, " %s", b.Repository)
|
||||
if b.Ref != "" {
|
||||
fmt.Fprintf(w, " at %s", b.Ref)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
for _, made := range b.Made {
|
||||
fmt.Fprintf(w, " %-10s %s\n", made.Kind, made.Reference)
|
||||
}
|
||||
// What it published and under what reuse key (novox/hq ADR 0408 §1).
|
||||
if p := b.Published; p != nil {
|
||||
if p.Artifact != "" {
|
||||
fmt.Fprintf(w, " published %s\n", p.Artifact)
|
||||
}
|
||||
if p.Package != nil {
|
||||
fmt.Fprintf(w, " published %s@%s\n", p.Package.Name, p.Package.Version)
|
||||
}
|
||||
}
|
||||
if b.ReuseKey != "" {
|
||||
fmt.Fprintf(w, " reuse key %s\n", b.ReuseKey)
|
||||
}
|
||||
if b.SourceFingerprint != "" {
|
||||
fmt.Fprintf(w, " source %s\n", b.SourceFingerprint)
|
||||
}
|
||||
if !b.Worked() {
|
||||
// The builder's own first line. The whole failure is often a build log, and printing
|
||||
// it here would bury every other row.
|
||||
fmt.Fprintf(w, " %s\n", firstLine(b.Failed))
|
||||
}
|
||||
}
|
||||
|
||||
// builderCommand issues a build machine its own broker credential.
|
||||
//
|
||||
// **A build machine is not a node**, and giving it a node's account would let it read another
|
||||
@@ -459,7 +484,7 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa
|
||||
return "", err
|
||||
}
|
||||
// Through a verb, only a repository the catalogue builds from (novox/hq ADR 0266).
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
if err := verbMayAsk(ctx, source, repository, path); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -798,7 +823,7 @@ func buildAndShow(ctx context.Context, source buildSource, path, ref string, wai
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := verbMayAsk(ctx, source, repository); err != nil {
|
||||
if err := verbMayAsk(ctx, source, repository, path); err != nil {
|
||||
return err
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
|
||||
@@ -389,10 +389,31 @@ func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, o
|
||||
// least one required status, no administrator merging past one.
|
||||
//
|
||||
// Anything else only when the build request was kept as asked at the controller's terminal, for this very
|
||||
// repository and path. path is the module's directory as built; branch the trunk it is registered from.
|
||||
// repository and path; or, for the build of a merge, when mesh-delivery announced it on the operator's Approve of
|
||||
// that head (novox/hq ADR 0420 §7 for a moved module, ADR 0432 for a new one), the forge's checks still holding. path is the module's directory as built; branch the trunk it is registered from.
|
||||
func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source,
|
||||
buildID, path, branch string) (int64, error) {
|
||||
forms := newSourceForms(ctx, inv)
|
||||
return mayRegisterIn(ctx, inv, newSourceForms(ctx, inv), module, built, buildID, path, branch)
|
||||
}
|
||||
|
||||
// catalogueReader is the catalogue's manifests, as the router's records are found in them.
|
||||
type catalogueReader interface {
|
||||
Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error)
|
||||
}
|
||||
|
||||
// registrations is what a registration's decision reads of the mesh: *inventory.Inventory, or a test's stand-in.
|
||||
type registrations interface {
|
||||
catalogueReader
|
||||
SourceOf(ctx context.Context, module string) (inventory.Source, error)
|
||||
SourceIdentity(ctx context.Context, module string) (int64, error)
|
||||
Catalogued(ctx context.Context) ([]inventory.Entry, error)
|
||||
BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error)
|
||||
MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error)
|
||||
}
|
||||
|
||||
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
||||
func mayRegisterIn(ctx context.Context, inv registrations, forms *sourceForms, module string, built inventory.Source,
|
||||
buildID, path, branch string) (int64, error) {
|
||||
was, err := inv.SourceOf(ctx, module)
|
||||
isNew := errors.Is(err, inventory.ErrNoSuchModule)
|
||||
if err != nil && !isNew {
|
||||
@@ -404,7 +425,8 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
}
|
||||
refuse := func(why string) (int64, error) {
|
||||
return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+
|
||||
"controller's terminal, never through a verb: a registered module is what the next push sends, and whoever "+
|
||||
"controller's terminal, or by the build of a merge mesh-delivery announced on the operator's Approve of that "+
|
||||
"head, never through a verb: a registered module is what the next push sends, and whoever "+
|
||||
"may call a verb includes agents (novox/hq ADR 0266)", errNotItsSource, why)
|
||||
}
|
||||
|
||||
@@ -438,8 +460,11 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
// **Or the move the operator approved** (novox/hq ADR 0420 §7, issue 1025): a known module built from another
|
||||
// repository by a merge, whose pull request mesh-delivery merged on the operator's Approve of that head.
|
||||
// A verb alone, or a merge with no such warrant, is refused as before.
|
||||
// **Or a new module from a repository nothing registered is built from, on the same warrant** (novox/hq ADR 0432,
|
||||
// issue 1146): the build of a merge mesh-delivery announced on the operator's Approve of that head registers it.
|
||||
// The forge's protection and the repository's identity are checked below as for every registration.
|
||||
movedWhy := ""
|
||||
if why != "" && !terminal && !isNew {
|
||||
if why != "" && !terminal {
|
||||
accepted, err := approvedMove(ctx, inv, forms, buildID, built)
|
||||
switch {
|
||||
case err == nil:
|
||||
@@ -507,7 +532,10 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
owner, name, m, facts.ID, m, id))
|
||||
}
|
||||
}
|
||||
if movedWhy != "" {
|
||||
if movedWhy != "" && isNew {
|
||||
fmt.Printf("%s: %s is new, from %s/%s — registered at the operator's Approve: %s\n", buildID, module, owner, name,
|
||||
movedWhy)
|
||||
} else if movedWhy != "" {
|
||||
fmt.Printf("%s: %s moved from %s to %s/%s — registered at the operator's Approve: %s\n", buildID, module,
|
||||
sourceWords(was.Repository, was.Seat), owner, name, movedWhy)
|
||||
}
|
||||
@@ -516,7 +544,7 @@ func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module strin
|
||||
|
||||
// askedHereFor says the build was asked at the controller's terminal, for this repository and this path: a kept
|
||||
// request marked so, whose source is the outcome's (novox/hq ADR 0266).
|
||||
func askedHereFor(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
||||
func askedHereFor(ctx context.Context, inv registrations, forms *sourceForms, buildID string,
|
||||
built inventory.Source, path string) (bool, error) {
|
||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||
if err != nil || !found || !r.AtTerminal {
|
||||
@@ -539,13 +567,49 @@ func sourceWords(repository, seat string) string {
|
||||
return buildSource{Repository: repository, Seat: seat}.String()
|
||||
}
|
||||
|
||||
// firstBuildWarrant is what the controller checked before it asks the first build of the modules a merge adds to a
|
||||
// repository nothing is built from: the merge was made on the operator's Approve of its head (approvedAnnounced). It
|
||||
// travels in the context of that ask alone, in this process, so no verb carries one.
|
||||
type firstBuildWarrant struct {
|
||||
repository string // owner/name on the git seat
|
||||
paths map[string]bool
|
||||
why string
|
||||
}
|
||||
|
||||
type firstBuildWarrantKey struct{}
|
||||
|
||||
// withFirstBuildWarrant is ctx carrying the warrant for asking the first builds of paths in repository.
|
||||
func withFirstBuildWarrant(ctx context.Context, repository string, paths []string, why string) context.Context {
|
||||
w := &firstBuildWarrant{repository: repository, paths: map[string]bool{}, why: why}
|
||||
for _, p := range paths {
|
||||
w.paths[strings.Trim(p, "/")] = true
|
||||
}
|
||||
return context.WithValue(ctx, firstBuildWarrantKey{}, w)
|
||||
}
|
||||
|
||||
func firstBuildWarrantIn(ctx context.Context) *firstBuildWarrant {
|
||||
w, _ := ctx.Value(firstBuildWarrantKey{}).(*firstBuildWarrant)
|
||||
return w
|
||||
}
|
||||
|
||||
// covers says the warrant is for exactly this ask: the repository on the git seat, and a module path the merge holds.
|
||||
func (w *firstBuildWarrant) covers(source buildSource, path string) bool {
|
||||
return w != nil && source.Seat == gitSeat && strings.EqualFold(strings.Trim(source.Repository, "/"), w.repository) &&
|
||||
w.paths[strings.Trim(path, "/")]
|
||||
}
|
||||
|
||||
// verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from
|
||||
// (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be
|
||||
// registered anyway. url is the repository as it is cloned. At the terminal anything may be asked.
|
||||
func verbMayAsk(ctx context.Context, source buildSource, url string) error {
|
||||
func verbMayAsk(ctx context.Context, source buildSource, url, path string) error {
|
||||
if startedAtTheTerminal() {
|
||||
return nil
|
||||
}
|
||||
// **Or the first build of a module a merge adds on the operator's Approve** (novox/hq ADR 0432, issue 1146): the
|
||||
// controller's own ask, carrying the warrant it checked, for that repository and the module paths the merge holds.
|
||||
if firstBuildWarrantIn(ctx).covers(source, path) {
|
||||
return nil
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -134,18 +134,18 @@ func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T)
|
||||
ctx := t.Context()
|
||||
t.Setenv(verbVar, "build")
|
||||
t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat")
|
||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git")
|
||||
err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git", "")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("a verb's build of an agent's repository was asked: %v", err)
|
||||
}
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"},
|
||||
"http://forge.internal:20000/novox/mesh-catalog.git"); err != nil {
|
||||
"http://forge.internal:20000/novox/mesh-catalog.git", ""); err != nil {
|
||||
t.Fatalf("a verb's build of the catalogue repository was refused: %v", err)
|
||||
}
|
||||
t.Setenv(verbVar, "")
|
||||
t.Setenv(link.CallerVar, "")
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil {
|
||||
if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "", ""); err != nil {
|
||||
t.Fatalf("the terminal was refused: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -317,7 +317,7 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
||||
keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil
|
||||
}
|
||||
routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
routerAskOf = func(_ context.Context, _ catalogueReader, _, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
if id != a.merge.AskID {
|
||||
t.Errorf("%s: the router was asked for %q", c.what, id)
|
||||
}
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
@@ -32,3 +37,43 @@ func TestABuildIDSaysWhenItWasAsked(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 1204: builds says the source fingerprint the builder recorded, the one tool that reads it.
|
||||
func TestBuildsSaysTheSourceFingerprint(t *testing.T) {
|
||||
at := time.Date(2026, 10, 19, 10, 40, 0, 0, time.UTC)
|
||||
built := inventory.Build{ID: "build-1", Module: "mailu", Commit: "b4527d9091e972f28cb365623d9280045e7741a3", On: "ace",
|
||||
Repository: "novox/mesh-catalog", At: at, ReuseKey: "r1:abc",
|
||||
SourceFingerprint: "src1:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"}
|
||||
|
||||
// 1204/AC1
|
||||
var out bytes.Buffer
|
||||
printBuild(&out, built)
|
||||
lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n")
|
||||
if last := lines[len(lines)-1]; last != " source "+built.SourceFingerprint {
|
||||
t.Errorf("last line %q; want the source line after the reuse key:\n%s", last, out.String())
|
||||
}
|
||||
if !strings.Contains(lines[len(lines)-2], "reuse key r1:abc") {
|
||||
t.Errorf("the source line does not follow the reuse key:\n%s", out.String())
|
||||
}
|
||||
|
||||
// 1204/AC2
|
||||
without := built
|
||||
without.SourceFingerprint = ""
|
||||
var plain bytes.Buffer
|
||||
printBuild(&plain, without)
|
||||
if strings.Contains(plain.String(), "source") {
|
||||
t.Errorf("a build with no fingerprint says a source line:\n%s", plain.String())
|
||||
}
|
||||
if want := strings.Join(lines[:len(lines)-1], "\n") + "\n"; plain.String() != want {
|
||||
t.Errorf("the other lines changed:\n%s\nwant\n%s", plain.String(), want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1215/AC4: what the builder said of the inputs is what the controller keeps of the build.
|
||||
func TestTheControllerKeepsTheInputsABuilderSaid(t *testing.T) {
|
||||
in := &sdkdelivery.SourceInputs{Module: "mailu", Tree: "t", Toolchains: map[string]string{"code": "ts sha256:aa 01"}}
|
||||
kept := buildFrom(link.BuildResult{ID: "build-1", Repository: "r", Commit: "c", SourceFingerprint: "src1:x", SourceInputs: in})
|
||||
if !reflect.DeepEqual(kept.SourceInputs, in) {
|
||||
t.Errorf("the controller keeps %+v, the builder said %+v", kept.SourceInputs, in)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The chain on the bus (novox/hq ADR 0408, design 47 milestone 5): builds, rollouts and assignments are keys of the
|
||||
// bus's key-value store, written by the controller that holds the lease, and the controller's store keeps none of them.
|
||||
|
||||
// chainHolds is whether this process may write the chain now: the lease's gate, the one every act passes.
|
||||
func chainHolds(ctx context.Context) error {
|
||||
_, err := theLease.epoch(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// openTheChain opens the chain's buckets on the bus: the serving controller's connection, or one dialled for a command.
|
||||
func openTheChain(ctx context.Context) (*chain.Chain, error) {
|
||||
js, err := aBus()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Not closed: a borrowed connection closes nothing, and a dialled one lasts as long as the command.
|
||||
return chain.OnTheBus(ctx, js.Conn(), chainHolds)
|
||||
}
|
||||
|
||||
// attachTheChain gives the inventory of the serving controller the chain on its connection.
|
||||
func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.Server) error {
|
||||
bus, ok := server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
return errors.New("the chain lives on the bus, and this control plane is not on it")
|
||||
}
|
||||
c, err := chain.OnTheBus(ctx, bus.Conn, chainHolds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inv.UseChain(c)
|
||||
inv.RequireLayout()
|
||||
// And the builds laid out under their modules, once, before they are read (issue 1121) — **before this returns**, so
|
||||
// nothing the start goes on to read meets a chain not laid out yet: on 2026-10-18 the layout ran beside the start, a
|
||||
// read of the start was refused, and the controller exited and was started again every five seconds. The layout of
|
||||
// keys laid out before skips them, so this takes seconds; one that fails is tried again beside the serving, loudly.
|
||||
if err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) }); err != nil {
|
||||
fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err)
|
||||
go layBuildsOutSoon(ctx, inv)
|
||||
}
|
||||
// And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving.
|
||||
go askAgainSoon(ctx, inv)
|
||||
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
||||
go copySentBuildsSoon(ctx, inv)
|
||||
return nil
|
||||
}
|
||||
|
||||
// askAgain asks once more for each build request the move found never handed over (novox/hq issue 981 note 7 Q5): the
|
||||
// requests a merge could not ask for, which a person's hand `build` would otherwise have to repeat. **At most once**: a
|
||||
// request is claimed before it is asked and settled after, so a crash between loses the ask, which is where it stood.
|
||||
func askAgain(ctx context.Context, inv *inventory.Inventory) {
|
||||
pending, err := inv.RequestsToAskAgain(ctx)
|
||||
if err != nil {
|
||||
fmt.Printf("chain: the requests never handed over cannot be read: %v\n", err)
|
||||
return
|
||||
}
|
||||
for _, r := range pending {
|
||||
if claimed, err := inv.ClaimToAskAgain(ctx, r.ID); err != nil || !claimed {
|
||||
if err != nil {
|
||||
fmt.Printf("chain: %s cannot be claimed to be asked again: %v\n", r.ID, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
id, err := askABuild(ctx, buildSource{Repository: r.Repository, Seat: r.Seat}, r.Path, r.Ref)
|
||||
if err != nil {
|
||||
if serr := inv.AskedAgainAs(ctx, r.ID, "not asked again: "+err.Error()); serr != nil {
|
||||
fmt.Printf("chain: %s was not asked again, and that could not be kept: %v\n", r.ID, serr)
|
||||
}
|
||||
fmt.Printf("chain: %s was never handed over and cannot be asked again: %v\n", r.ID, err)
|
||||
continue
|
||||
}
|
||||
recordBuildRequest(ctx, inv, inventory.BuildRequest{ID: id, Repository: r.Repository, Seat: r.Seat, Path: r.Path,
|
||||
Ref: r.Ref, Commit: r.Commit, For: r.For})
|
||||
if err := inv.AskedAgainAs(ctx, r.ID, id); err != nil {
|
||||
fmt.Printf("chain: %s was asked again as %s, and that could not be kept: %v\n", r.ID, id, err)
|
||||
}
|
||||
fmt.Printf("chain: %s, never handed over, is asked again as %s\n", r.ID, id)
|
||||
}
|
||||
}
|
||||
|
||||
// askAgainSoon waits for the controller to be serving, which an ask needs, and then asks again.
|
||||
func askAgainSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Minute):
|
||||
askAgain(ctx, inv)
|
||||
}
|
||||
}
|
||||
|
||||
// kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not
|
||||
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
||||
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
||||
|
||||
// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
|
||||
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
|
||||
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
|
||||
// that names a commit and empties each column it has copied.
|
||||
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
failed := false
|
||||
for {
|
||||
_, err := inv.CopySentBuilds(ctx)
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
|
||||
Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
|
||||
if err != nil {
|
||||
fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
|
||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||
o.Summary += err.Error()
|
||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if keeper := conditionsFrom; keeper != nil && failed {
|
||||
_, _ = keeper.Clear(ctx, o.Key(), "copied")
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(chainMoveRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// chainMoveRetry is how long the copy waits before it tries again.
|
||||
const chainMoveRetry = 30 * time.Second
|
||||
|
||||
// kindBuildsNotLaidOut is the condition of the layout of the builds under their modules that has not succeeded: until it
|
||||
// has, no build is read, and a send that needs one is refused with the reason.
|
||||
const kindBuildsNotLaidOut = "builds-not-laid-out"
|
||||
|
||||
// layBuildsOutSoon lays the builds out under their modules as soon as this controller holds the lease (the chain refuses a
|
||||
// write before), and tries again every half minute while it fails, loudly: each failure is printed, and the first raises an
|
||||
// urgent condition until it has worked. It is idempotent (LayBuildsOut).
|
||||
func layBuildsOutSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
failed := false
|
||||
for {
|
||||
err := inv.LayBuildsOut(ctx, func(format string, args ...any) { fmt.Printf("chain: "+format+"\n", args...) })
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindBuildsNotLaidOut, Token: kindBuildsNotLaidOut,
|
||||
Kind: kindBuildsNotLaidOut, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Source: "chain", Summary: "the builds are not laid out under their modules, so none is read: "}
|
||||
if err != nil {
|
||||
fmt.Printf("chain: the builds are not laid out under their modules, and none is read until they are: %v\n", err)
|
||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||
o.Summary += err.Error()
|
||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if keeper := conditionsFrom; keeper != nil && failed {
|
||||
_, _ = keeper.Clear(ctx, o.Key(), "laid out")
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(chainMoveRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/lease"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **Acceptance 10 of design 47: one read of an assignment names the commit, the build and the rollout that put it there,
|
||||
// and one read of the rollout lists the assignments it changed** (novox/hq ADR 0408, issue 1095). It enters by the live
|
||||
// trigger — a merge heard, its batch cut into a plan when the window closed, its build answered through the build seat's
|
||||
// path, the plan advanced — and writes no key and no row by hand. Only the machine's end of the send is the test's: the send
|
||||
// is recorded as the controller records one it made, and the gate's facts are the store's.
|
||||
func TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var sent [][]string
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
sent = append(sent, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
digest := fmt.Sprintf("d-%s-%d", node, len(sent))
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest,
|
||||
map[string]string{"notes": current["notes"].Commit}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, err := open.inventory.LastReports(ctx)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
|
||||
// A merge of notes is heard, and its window closes: the batch is cut into the walk.
|
||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(time.Second+90*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
||||
}
|
||||
rollout := ws[0].ID
|
||||
|
||||
// The walk asks its first tier (at the cut, or at its next step); the build seat answers, and the controller takes the
|
||||
// outcome in as it does.
|
||||
asking, err := inv.PlanByID(ctx, rollout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ {
|
||||
advancePlans(ctx, open)
|
||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
buildID := asking.Modules["notes"].Build
|
||||
if buildID == "" || len(*asked) != 1 || !strings.HasSuffix((*asked)[0], "#"+buildID) {
|
||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("a", 64)
|
||||
result := outcome(buildID, "modules/notes", "c-notes", "notes", "")
|
||||
result.ReuseKey = "rk-notes"
|
||||
result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}}
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The walk sends the tier to its first machine.
|
||||
for try := 0; try < 3 && len(sent) == 0; try++ {
|
||||
advancePlans(ctx, open)
|
||||
}
|
||||
if !slices.Equal(slices.Concat(sent...), []string{"anchor"}) {
|
||||
t.Fatalf("sent %v", sent)
|
||||
}
|
||||
|
||||
// One read of the assignment.
|
||||
on, err := inv.AssignmentsOn(ctx, "anchor")
|
||||
a, ok := assignmentOf(on, "notes")
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("the assignments of anchor: %+v %v", on, err)
|
||||
}
|
||||
// Beside it a module pulled in by the assignment, which no send of this walk put anywhere.
|
||||
if wireguard, pulled := assignmentOf(on, "mesh-wireguard"); pulled && (wireguard.Build != "" || wireguard.Rollout != "") {
|
||||
t.Fatalf("a module this rollout did not send names a build or a rollout: %+v", wireguard)
|
||||
}
|
||||
if a.Module != "notes" || a.Commit != "c-notes" || a.Build != buildID || a.Rollout != rollout || a.Artifact != digest ||
|
||||
a.Since.IsZero() || a.Assigned.IsZero() || !a.Stands() {
|
||||
t.Fatalf("the assignment names %+v, not commit c-notes, build %s, artifact %s, rollout %s", a, buildID, digest, rollout)
|
||||
}
|
||||
// One read of the rollout.
|
||||
p, err := inv.PlanByID(ctx, rollout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantChange := chain.Change{Node: "anchor", Module: "notes", After: chain.Ref{Build: buildID, Commit: "c-notes"}}
|
||||
if !slices.Contains(p.Changed, wantChange) || !slices.Contains(p.Builds, buildID) || len(p.Steps) != 1 ||
|
||||
p.Steps[0].Node != "anchor" || p.Steps[0].Sent == nil {
|
||||
t.Fatalf("the rollout lists changed %+v, builds %v, steps %+v; want %+v", p.Changed, p.Builds, p.Steps, wantChange)
|
||||
}
|
||||
// And one read of the build names the same commit, with what it published and its reuse key.
|
||||
built, found, err := inv.BuildByID(ctx, buildID)
|
||||
if err != nil || !found || built.Commit != "c-notes" || built.Module != "notes" {
|
||||
t.Fatalf("the build: %+v %v %v", built, found, err)
|
||||
}
|
||||
lately, err := inv.Builds(ctx, "notes", 1)
|
||||
if err != nil || len(lately) != 1 || lately[0].ReuseKey != "rk-notes" || lately[0].Published == nil ||
|
||||
lately[0].Published.Artifact != digest {
|
||||
t.Fatalf("what the build published: %+v %v", lately, err)
|
||||
}
|
||||
// The verbs say the links.
|
||||
if lines := changedLines(p); len(lines) != 2 || !strings.Contains(lines[1], "anchor/notes") ||
|
||||
!strings.Contains(lines[1], buildID) {
|
||||
t.Fatalf("`plans` says %v", lines)
|
||||
}
|
||||
if lines := runLines(on); len(lines) != 2 || !strings.Contains(lines[1], buildID) || !strings.Contains(lines[1], rollout) {
|
||||
t.Fatalf("`node show` says %v", lines)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine sent again what it runs says nothing new, and a push by hand with no rollout fills what is missing and blanks
|
||||
// nothing (issue 1095).
|
||||
func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-1", Module: "notes", Commit: "c1", Repository: "novox/notes"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
node := nodeID(t, open, "anchor")
|
||||
inRollout := inventory.WithRollout(ctx, "plan-1", map[string]inventory.RolloutModule{"notes": {Commit: "c1", Build: "build-1"}})
|
||||
if err := inv.RecordSent(inRollout, node, "d1", map[string]string{"notes": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Sent again by hand, with no rollout: the assignment still names the rollout that put the build there.
|
||||
if err := inv.RecordSent(ctx, node, "d2", map[string]string{"notes": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
on, _ := inv.AssignmentsOn(ctx, "anchor")
|
||||
if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" {
|
||||
t.Fatalf("the assignment names %+v", on)
|
||||
}
|
||||
// A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit.
|
||||
if err := inv.RecordSent(ctx, node, "d3", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() {
|
||||
t.Fatalf("a send of unknown builds left %+v", on)
|
||||
}
|
||||
// A module sent a commit no build is kept of is named by its commit alone.
|
||||
if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c2" || runOf(on, "notes").Build != "" ||
|
||||
runOf(on, "notes").Rollout != "" {
|
||||
t.Fatalf("the assignment after a push of c2 by hand names %+v", on)
|
||||
}
|
||||
}
|
||||
|
||||
// assignmentOf is one module's assignment among those of a machine: assigning a module assigns what it requires beside it.
|
||||
func assignmentOf(on []chain.Assignment, module string) (chain.Assignment, bool) {
|
||||
for _, a := range on {
|
||||
if a.Module == module {
|
||||
return a, true
|
||||
}
|
||||
}
|
||||
return chain.Assignment{}, false
|
||||
}
|
||||
|
||||
func runOf(on []chain.Assignment, module string) chain.Assignment {
|
||||
a, _ := assignmentOf(on, module)
|
||||
return a
|
||||
}
|
||||
|
||||
// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered
|
||||
// through the `assign` verb for a module whose build is in flight, one read of `<node>.<module>` says state pending with the
|
||||
// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied,
|
||||
// and the table of pending assignments is read by nobody. No key is written by hand.
|
||||
func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) {
|
||||
open := aCatalogueMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
asksWithPaths(t)
|
||||
mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
|
||||
if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
|
||||
t.Fatalf("an assignment while its build runs was refused: %v", err)
|
||||
}
|
||||
waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
||||
if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() ||
|
||||
waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" ||
|
||||
!strings.HasPrefix(waiting.Pending.ID, "p-") {
|
||||
t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err)
|
||||
}
|
||||
if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") {
|
||||
t.Fatalf("a module only waiting is assigned: %v", got)
|
||||
}
|
||||
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors")
|
||||
if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil ||
|
||||
made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID {
|
||||
t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err)
|
||||
}
|
||||
if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
|
||||
t.Fatal("the build registered sensors and it was not assigned")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a
|
||||
// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of
|
||||
// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the
|
||||
// build before and after.
|
||||
|
||||
// rolloutModules is what a plan sends of each module: the commit and the build of it.
|
||||
func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule {
|
||||
out := make(map[string]inventory.RolloutModule, len(p.Modules))
|
||||
for m, s := range p.Modules {
|
||||
if s != nil && s.Commit != "" {
|
||||
out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout.
|
||||
func sendingAs(ctx context.Context, p *inventory.Plan) context.Context {
|
||||
return inventory.WithRollout(ctx, p.ID, rolloutModules(p))
|
||||
}
|
||||
|
||||
// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit,
|
||||
// else the commit the machine was last sent for it. Read before the send, which writes the new ones.
|
||||
func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref {
|
||||
was := map[string]chain.Ref{}
|
||||
if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known {
|
||||
for m, commit := range sent {
|
||||
was[m] = chain.Ref{Commit: commit}
|
||||
}
|
||||
}
|
||||
if ran, err := inv.AssignmentRuns(ctx, node); err == nil {
|
||||
for m, r := range ran {
|
||||
was[m] = r
|
||||
}
|
||||
}
|
||||
return was
|
||||
}
|
||||
|
||||
// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and
|
||||
// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the
|
||||
// module is replaced, so a send made again says it once.
|
||||
func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) {
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}}
|
||||
if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 {
|
||||
change.Before = p.Changed[at].Before
|
||||
p.Changed[at] = change
|
||||
} else {
|
||||
p.Changed = append(p.Changed, change)
|
||||
}
|
||||
if s.Build != "" && !slices.Contains(p.Builds, s.Build) {
|
||||
p.Builds = append(p.Builds, s.Build)
|
||||
}
|
||||
}
|
||||
if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) {
|
||||
sent := at.UTC()
|
||||
p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent})
|
||||
}
|
||||
}
|
||||
|
||||
// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none.
|
||||
func refSaid(r chain.Ref) string {
|
||||
switch {
|
||||
case r.Build == "" && r.Commit == "":
|
||||
return "nothing"
|
||||
case r.Build == "":
|
||||
return "commit " + short(r.Commit)
|
||||
}
|
||||
return r.Build + " (" + short(r.Commit) + ")"
|
||||
}
|
||||
|
||||
// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after.
|
||||
func changedLines(p inventory.Plan) []string {
|
||||
if len(p.Changed) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := []string{" changed:"}
|
||||
for _, c := range p.Changed {
|
||||
out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the
|
||||
// rollout that put it there and since when.
|
||||
func runLines(assigned []chain.Assignment) []string {
|
||||
var out []string
|
||||
for _, a := range assigned {
|
||||
if a.Build == "" && a.Commit == "" {
|
||||
continue
|
||||
}
|
||||
line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit}))
|
||||
if a.Rollout != "" {
|
||||
line += ", sent by " + a.Rollout
|
||||
}
|
||||
if !a.Since.IsZero() {
|
||||
line += " at " + a.Since.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]string{" what they run, as the chain names it:"}, out...)
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one
|
||||
// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the
|
||||
// rollout carried once each, and one step for a machine and a tier.
|
||||
func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{
|
||||
"notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}}
|
||||
was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}}
|
||||
|
||||
noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at)
|
||||
noteChanges(&p, "laptop", []string{"notes"}, nil, at)
|
||||
// The send noted again, after what was sent had become what the machine runs: the first before stands.
|
||||
noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute))
|
||||
|
||||
want := []chain.Change{
|
||||
{Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
{Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}},
|
||||
{Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
}
|
||||
if len(p.Changed) != len(want) {
|
||||
t.Fatalf("changed %+v", p.Changed)
|
||||
}
|
||||
for k := range want {
|
||||
if p.Changed[k] != want[k] {
|
||||
t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k])
|
||||
}
|
||||
}
|
||||
if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" {
|
||||
t.Errorf("builds %v", p.Builds)
|
||||
}
|
||||
if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" {
|
||||
t.Errorf("steps %+v", p.Steps)
|
||||
}
|
||||
lines := changedLines(p)
|
||||
if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") ||
|
||||
!strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") {
|
||||
t.Errorf("changed lines %q", lines)
|
||||
}
|
||||
if got := changedLines(inventory.Plan{}); got != nil {
|
||||
t.Errorf("a plan that changed nothing says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
lines := runLines([]chain.Assignment{
|
||||
{Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at},
|
||||
{Node: "anchor", Module: "web", Commit: "c3"},
|
||||
{Node: "anchor", Module: "never-sent"},
|
||||
})
|
||||
if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") ||
|
||||
!strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") {
|
||||
t.Fatalf("lines %q", lines)
|
||||
}
|
||||
if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil {
|
||||
t.Fatal("a module never sent is said to run something")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR
|
||||
// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's
|
||||
// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the
|
||||
// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays.
|
||||
func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asked := asksRecorded(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second))
|
||||
cutAt(t, open, t0.Add(91*time.Second))
|
||||
ws, _ := walks(t, open)
|
||||
if len(ws) != 1 {
|
||||
t.Fatalf("%d walk(s) after the cut", len(ws))
|
||||
}
|
||||
rollout := ws[0].ID
|
||||
asking, err := inv.PlanByID(ctx, rollout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for try := 0; try < 3 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == ""); try++ {
|
||||
advancePlans(ctx, open)
|
||||
if asking, err = inv.PlanByID(ctx, rollout); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
buildID := asking.Modules["notes"].Build
|
||||
if buildID == "" || len(*asked) != 1 {
|
||||
t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("b", 64)
|
||||
result := outcome(buildID, "modules/notes", "c-notes", "notes", "")
|
||||
result.Made = []link.MadeArtifact{{Name: "notes", Kind: "image", Reference: "artifact-store://notes/notes@" + digest}}
|
||||
if err := (builds{inv: inv, open: open}).Built(ctx, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// What the live path made is in the chain.
|
||||
if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" {
|
||||
t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err)
|
||||
}
|
||||
if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout {
|
||||
t.Fatalf("the rollout is not read from the chain: %+v %v", p, err)
|
||||
}
|
||||
if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, ok := assignmentOf(on, "notes"); !ok {
|
||||
t.Fatalf("the assignment is not read from the chain: %+v", on)
|
||||
}
|
||||
|
||||
// And the catalog of the store holds none of them.
|
||||
have := map[string]bool{}
|
||||
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
||||
have[strings.Trim(row[0], `"`)] = true
|
||||
}
|
||||
for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} {
|
||||
if have[table] {
|
||||
t.Errorf("the table %s is in the controller's store", table)
|
||||
}
|
||||
}
|
||||
for _, table := range []string{"build_gate", "assignment_generation"} {
|
||||
if !have[table] {
|
||||
t.Errorf("the table %s, which stays, is gone", table)
|
||||
}
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
|
||||
t.Errorf("the function raise_assignment_generation is in the store: %v", rows)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1212/AC1 and AC2: the check request names the pulls bucket of each module claiming the delivery seat, in the order
|
||||
// of the claimants' names.
|
||||
func TestACheckNamesThePullsBucketsOfTheSeatsClaimants(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
entries []inventory.Entry
|
||||
want []string
|
||||
}{
|
||||
"only the gate": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("mesh-delivery")}, []string{"mesh-delivery-gate_pulls"}},
|
||||
"only today's": {[]inventory.Entry{claimingTheSeat("mesh-delivery"), namedOnly("gitea")}, []string{"mesh-delivery_pulls"}},
|
||||
"both, name order": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls", "mesh-delivery-gate_pulls"}},
|
||||
"none": {[]inventory.Entry{namedOnly("gitea")}, nil},
|
||||
} {
|
||||
if got := pullsBucketsOf(c.entries); !reflect.DeepEqual(got, c.want) {
|
||||
t.Errorf("%s: %v, want %v", name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1212/AC1 and AC2: the check a request carries names those buckets (the wiring, not only the helper).
|
||||
func TestTheCheckRequestCarriesThePullsBuckets(t *testing.T) {
|
||||
for name, c := range map[string]struct {
|
||||
entries []inventory.Entry
|
||||
want []string
|
||||
}{
|
||||
"only the gate": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate")}, []string{"mesh-delivery-gate_pulls"}},
|
||||
"only today's": {[]inventory.Entry{claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls"}},
|
||||
"both, name order": {[]inventory.Entry{claimingTheSeat("mesh-delivery-gate"), claimingTheSeat("mesh-delivery")}, []string{"mesh-delivery_pulls", "mesh-delivery-gate_pulls"}},
|
||||
} {
|
||||
got := checkOf(link.PullUpdated{Owner: "novox", Repo: "a", Number: 7}, checkScope{}, c.entries, nil, nil, nil, nil)
|
||||
if !reflect.DeepEqual(got.Pulls, c.want) || got.Owner != "novox" || got.Number != 7 {
|
||||
t.Errorf("%s: %+v, want pulls %v", name, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -305,13 +306,19 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop
|
||||
Held: heldBy(ctx),
|
||||
Seats: seatBases(ctx),
|
||||
Source: sourceOnSeat(source),
|
||||
Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready,
|
||||
Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New,
|
||||
Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds),
|
||||
Running: runningBy(ctx)},
|
||||
Check: checkOf(p, scope, entries, beside, builds, namedForTheCheck(ctx, p, builds), runningBy(ctx)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// checkOf is the check a request carries: the pull request, what the change touches, and the buckets of the pull
|
||||
// requests' keys of the modules claiming the delivery seat (novox/hq ADR 0437 §3, issue 1212).
|
||||
func checkOf(p link.PullUpdated, scope checkScope, entries []inventory.Entry, beside map[string]link.CheckedOut,
|
||||
builds []link.CheckBuild, named []link.CheckedBuild, running map[string]string) *link.CheckRequest {
|
||||
return &link.CheckRequest{Pulls: pullsBucketsOf(entries), Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base,
|
||||
Trunk: p.BaseSHA, Ready: p.Ready, Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents,
|
||||
New: scope.New, Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: named, Running: running}
|
||||
}
|
||||
|
||||
// besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository
|
||||
// by the module the mesh builds from it, at besideRefs. running is the commit the mesh runs of a module.
|
||||
func besideOf(entries []inventory.Entry, running func(module string) string,
|
||||
@@ -661,3 +668,13 @@ func sayChecked(ctx context.Context, c link.Checked) {
|
||||
fmt.Printf("%s: the verdict could not be said, so the pull request is not told it: %v\n", c.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// pullsBucketsOf are the buckets of the pull requests' keys a check names for the build agent: one per module
|
||||
// claiming the delivery seat, in the order of their names (novox/hq ADR 0437 §3, issue 1212).
|
||||
func pullsBucketsOf(entries []inventory.Entry) []string {
|
||||
var out []string
|
||||
for _, m := range claimantsOf(entries, catalogue.DeliverySeat) {
|
||||
out = append(out, broker.BucketName(m, "pulls"))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -56,13 +56,19 @@ func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}})
|
||||
// Built from a commit, so that a send carries them: a holder is asked only of what the machine was sent (issue 1109).
|
||||
built := inventory.Source{BuiltFrom: "c1"}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "keeper", Version: "1",
|
||||
Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
house, err := catalogue.ParseManifest([]byte(houseManifest))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, house)
|
||||
if err := inv.RegisterModule(ctx, house, built); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "laptop", "house"); err == nil {
|
||||
t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up")
|
||||
}
|
||||
|
||||
@@ -42,13 +42,53 @@ import (
|
||||
// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery
|
||||
// go`, `delivery stop` and `delivery walks`.
|
||||
|
||||
// onTheControllersPath are the modules whose walk never waits for the delivery's owner, and what each is.
|
||||
// onTheControllersPath are the modules, by name, whose walk never waits for the delivery's owner, and what each is.
|
||||
// The delivery's owner is not among them: it is whichever module claims the seat (ownPathOf, novox/hq ADR 0437 §3).
|
||||
var onTheControllersPath = map[string]string{
|
||||
"mesh-controller": "the controller",
|
||||
"mesh-host": "the node-engine",
|
||||
"node-tools": "the node tools",
|
||||
"nats": "the bus",
|
||||
catalogue.DeliverySeat: "the delivery's owner",
|
||||
"mesh-controller": "the controller",
|
||||
"mesh-host": "the node-engine",
|
||||
"node-tools": "the node tools",
|
||||
"nats": "the bus",
|
||||
}
|
||||
|
||||
// claimantsOf are the modules among the catalogued entries that claim a seat, each once, in name order.
|
||||
func claimantsOf(entries []inventory.Entry, seat string) []string {
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if e.Manifest.ClaimsSeat(seat) && !slices.Contains(out, e.Manifest.Module) {
|
||||
out = append(out, e.Manifest.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// ownPathOf is the controller's own path among these entries: the modules by name, and the modules that claim
|
||||
// the delivery seat, which is how the delivery's owner is found (novox/hq ADR 0437 §3).
|
||||
func ownPathOf(entries []inventory.Entry) map[string]string {
|
||||
own := make(map[string]string, len(onTheControllersPath)+1)
|
||||
for m, what := range onTheControllersPath {
|
||||
own[m] = what
|
||||
}
|
||||
for _, m := range claimantsOf(entries, catalogue.DeliverySeat) {
|
||||
own[m] = "the delivery's owner"
|
||||
}
|
||||
return own
|
||||
}
|
||||
|
||||
// onTheOwnPath is whether one module, with its manifest at hand, is on the controller's own path.
|
||||
func onTheOwnPath(module string, m catalogue.Manifest) bool {
|
||||
_, named := onTheControllersPath[module]
|
||||
return named || m.ClaimsSeat(catalogue.DeliverySeat)
|
||||
}
|
||||
|
||||
// ownPathNow is ownPathOf the catalogue as it is.
|
||||
func ownPathNow(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) {
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ownPathOf(entries), nil
|
||||
}
|
||||
|
||||
// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a
|
||||
@@ -69,8 +109,9 @@ func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDeliv
|
||||
if !deliverySeatHeld(entries) {
|
||||
return nil
|
||||
}
|
||||
onPath := ownPathOf(entries)
|
||||
for _, m := range modules {
|
||||
if _, own := onTheControllersPath[m]; own {
|
||||
if _, own := onPath[m]; own {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -327,16 +368,17 @@ func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Pla
|
||||
// the stop's one act. The kind is read from what the merges moved; the stopped walk itself is no walk a cut folds.
|
||||
func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, stopping string, again []inventory.BatchedMerge,
|
||||
now time.Time) (inventory.Plan, []*inventory.Plan, []inventory.MergeMove, error) {
|
||||
own := false
|
||||
for _, n := range movesOf(again) {
|
||||
if _, on := onTheControllersPath[n]; on {
|
||||
own = true
|
||||
}
|
||||
}
|
||||
entries, read, edges, err := theGraph(ctx, inv)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
}
|
||||
own := false
|
||||
onPath := ownPathOf(entries)
|
||||
for _, n := range movesOf(again) {
|
||||
if _, on := onPath[n]; on {
|
||||
own = true
|
||||
}
|
||||
}
|
||||
joins, _, _, err := relatedBatches(ctx, inv, again, own, stopping, entries, read, edges)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, nil, nil, err
|
||||
@@ -763,6 +805,8 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated")
|
||||
removed := set.String("removed", "", "the files it deletes, comma-separated")
|
||||
membersJSON := set.String("members", "", "a group's members, as JSON")
|
||||
membersFile := set.String("members-file", "", "a file of a group's members, as JSON: the same as --members, for a list "+
|
||||
"too long for an argument (novox/hq issue 1272)")
|
||||
group := set.String("group", "", "a delivery group's id")
|
||||
ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+
|
||||
"(novox/hq ADR 0406 §3)")
|
||||
@@ -777,6 +821,13 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
return err
|
||||
}
|
||||
var members []orderMember
|
||||
if *membersFile != "" {
|
||||
raw, err := os.ReadFile(*membersFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--members-file: %w", err)
|
||||
}
|
||||
*membersJSON = string(raw)
|
||||
}
|
||||
if *membersJSON != "" {
|
||||
if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil {
|
||||
return fmt.Errorf("the members are not readable: %w", err)
|
||||
@@ -935,7 +986,7 @@ func deliveryCommand(ctx context.Context, args []string) error {
|
||||
walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv))
|
||||
}
|
||||
return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks,
|
||||
"own-path": sortedKeysOf(ownPathWords())})
|
||||
"own-path": sortedKeysOf(ownPathOf(entries))})
|
||||
}
|
||||
return fmt.Errorf("delivery %s: plan, order, check, go, stop, backups-again or walks", sub)
|
||||
}
|
||||
@@ -1006,9 +1057,6 @@ func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Ti
|
||||
return nil
|
||||
}
|
||||
|
||||
// ownPathWords is the controller's own path as words, for an answer.
|
||||
func ownPathWords() map[string]string { return onTheControllersPath }
|
||||
|
||||
// theGraph is what the planner reads.
|
||||
func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository,
|
||||
[]inventory.Edge, error) {
|
||||
@@ -1181,8 +1229,32 @@ func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest {
|
||||
// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and,
|
||||
// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what
|
||||
// the walk recorded.
|
||||
//
|
||||
// The plan the announcement is made from is a copy that shares nothing with the walk, taken before the goroutine
|
||||
// starts: the walk goes on writing its steps' maps and pointers while the announcement is marshalled and its
|
||||
// touches read (novox/hq issue 1260), and it says the plan as it was when this was called.
|
||||
func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
go publishPlanMoved(ctx, bus, facts, p)
|
||||
kept, err := copyOfPlan(p)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err)
|
||||
return
|
||||
}
|
||||
go publishPlanMoved(ctx, bus, facts, kept)
|
||||
}
|
||||
|
||||
// copyOfPlan is the plan with nothing shared with the original: its JSON round trip, which is all of it but what
|
||||
// the plan never keeps (WalkSaid), carried over.
|
||||
func copyOfPlan(p inventory.Plan) (inventory.Plan, error) {
|
||||
raw, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
var kept inventory.Plan
|
||||
if err := json.Unmarshal(raw, &kept); err != nil {
|
||||
return inventory.Plan{}, err
|
||||
}
|
||||
kept.WalkSaid = p.WalkSaid
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) {
|
||||
|
||||
@@ -55,6 +55,17 @@ type stalledLine struct {
|
||||
Bound string `json:"bound"`
|
||||
H2 string `json:"h2"`
|
||||
Says string `json:"says"`
|
||||
// Severity is "urgent" for a group held or half merged and a refused member (novox/hq issues 1275, 1278); empty
|
||||
// or anything else is a warning, as before.
|
||||
Severity string `json:"severity,omitempty"`
|
||||
}
|
||||
|
||||
// severity is the condition's severity a stalled line asks: urgent when it says so, a warning otherwise.
|
||||
func (l stalledLine) severity() conditions.Severity {
|
||||
if l.Severity == string(conditions.Urgent) {
|
||||
return conditions.Urgent
|
||||
}
|
||||
return conditions.Warning
|
||||
}
|
||||
|
||||
// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state —
|
||||
@@ -165,7 +176,7 @@ func stalledObservations(lines []stalledLine) []conditions.Observation {
|
||||
out := make([]conditions.Observation, 0, len(lines))
|
||||
for _, l := range lines {
|
||||
o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled,
|
||||
Severity: conditions.Warning,
|
||||
Severity: l.severity(),
|
||||
Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+
|
||||
"`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID),
|
||||
Said: fmt.Sprintf("%s for %s", l.State, l.For)}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
func severityRaisedFor(t *testing.T, line string) conditions.Severity {
|
||||
t.Helper()
|
||||
var l stalledLine
|
||||
if err := json.Unmarshal([]byte(line), &l); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := stalledObservations([]stalledLine{l})
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("%d conditions", len(got))
|
||||
}
|
||||
return got[0].Severity
|
||||
}
|
||||
|
||||
func TestAnUrgentStalledLineRaisesAnUrgentCondition(t *testing.T) { // 1278/AC1
|
||||
line := `{"id":"novox/mesh-delivery#5","state":"held","for":"3m0s","bound":"0s","h2":"close","says":"held","severity":"urgent"}`
|
||||
if got := severityRaisedFor(t, line); got != conditions.Urgent {
|
||||
t.Errorf("an urgent line raised %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStalledLineWithoutOrWithAnUnknownSeverityRaisesAWarning(t *testing.T) { // 1278/AC2
|
||||
for _, extra := range []string{``, `,"severity":""`, `,"severity":"critical"`, `,"severity":"warning"`} {
|
||||
line := `{"id":"novox/a#1","state":"unanswered","for":"3m0s","bound":"2m0s","h2":"close","says":"x"` + extra + `}`
|
||||
if got := severityRaisedFor(t, line); got != conditions.Warning {
|
||||
t.Errorf("the line %s raised %q", extra, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -120,6 +120,16 @@ var probeRegistry = []probe{
|
||||
{ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " +
|
||||
"`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239",
|
||||
Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries},
|
||||
// What the watchman says stopped moving (ADR 0433 §2, issue 1156): each key of its stuck bucket not moved, but a
|
||||
// node's presence, which the silent check takes.
|
||||
{ID: probeStuckID, Asserts: "no key the watchman says stopped moving is left unsaid: each key of its stuck bucket " +
|
||||
"not moved is a condition of the delivery scope until it says moved", From: "ADR 0433 §2, issue 1156",
|
||||
Kind: kindStuck, Phase: 3, run: probeStuck},
|
||||
// A first merge into a repository the mesh builds nothing from, kept waiting for mesh-delivery's word on the
|
||||
// operator's Approve (novox/hq ADR 0432, issue 1146): said past a bound, never only in a log line (ADR 0426 §3, §8).
|
||||
{ID: probeFirstBuildsID, Asserts: "no first merge into a repository the mesh builds nothing from waits more than ten " +
|
||||
"minutes for mesh-delivery's announcement of the operator's Approve of its head", From: "ADR 0432, issue 1146",
|
||||
Kind: kindFirstBuildUnasked, Phase: 3, run: probeFirstBuilds},
|
||||
// The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget,
|
||||
// read from mesh-delivery's `times`; a measurement said, never a delivery held.
|
||||
{ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " +
|
||||
@@ -178,6 +188,10 @@ var probeRegistry = []probe{
|
||||
{ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " +
|
||||
"answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1,
|
||||
run: probeBackupHeld},
|
||||
// A plan's send key the bus refused (ADR 0433 §4, issue 1157): in place of the timer that stood in for it.
|
||||
{ID: sendKeyRefusedID, Asserts: "no delivery plan's key that watches a machine for its report was refused by the " +
|
||||
"bus at the newest look", From: "ADR 0433 §4, issue 1157", Kind: kindSendKeyRefused, Phase: 1,
|
||||
run: probeSendKeyRefused},
|
||||
}
|
||||
|
||||
// probeVerdict is one probe's outcome in a run.
|
||||
|
||||
@@ -72,8 +72,6 @@ func summarise(ds []inventory.Duration) []durationSummary {
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
case inventory.DurationHeartbeatGap:
|
||||
s.Suggests = "S1 bound 3×interval ≈ " + round(3*at(0.5))
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
@@ -138,7 +136,7 @@ func durationsCommand(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, heartbeat-gap, "+
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/identity"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// aForgeAndABuildSeat gives a test what a real ask of a build needs: the forge holding the git seat, assigned to anchor, so
|
||||
// the clone URL composes; the identity store; the build seat's queue on a real bus; and the environment of the serving controller, which is
|
||||
// never the controller's terminal (a verb's, in effect: startedAtTheTerminal is false).
|
||||
func aForgeAndABuildSeat(t *testing.T, open *stores) *broker.JetStream {
|
||||
t.Helper()
|
||||
js, _ := aBuildQueue(t)
|
||||
// The ask signs what it sends with the controller's identity, as the serving controller does.
|
||||
identity.ForTest(t)
|
||||
forge, err := catalogue.ParseManifest([]byte(`{"module":"gitea","version":"1",
|
||||
"claims":[{"name":"git","scope":"mesh"}],"provides":[{"name":"git","scope":"mesh"}],
|
||||
"serves":{"git":{"scheme":"http","port":3000}}}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
register(t, open, forge)
|
||||
if _, err := assign(t.Context(), open, "anchor", "gitea"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv(servedVar, "1")
|
||||
return js
|
||||
}
|
||||
|
||||
// 1146/AC9, through the real ask (askABuild, buildOneAsked, verbMayAsk) and a real bus: the first build of a module a
|
||||
// merge adds on the operator's Approve is queued on the build seat; the same ask without that warrant, and the warrant's
|
||||
// ask for a path the merge does not hold, are refused by the guard of a build through a verb.
|
||||
func TestTheApprovedFirstBuildPassesTheGuardOfAVerbAndIsQueued(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
js := aForgeAndABuildSeat(t, open)
|
||||
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
|
||||
// Without the warrant the guard refuses, whatever the repository holds.
|
||||
_, err := askABuild(ctx, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", "main")
|
||||
var policy *heldAtTheTerminal
|
||||
if !errors.As(err, &policy) {
|
||||
t.Fatalf("1146/AC9: a first build asked without the warrant was not refused: %v", err)
|
||||
}
|
||||
// And a path the merge does not hold is refused on the warrant.
|
||||
warranted := withFirstBuildWarrant(ctx, "novox/mesh-delivery", []string{""}, "the Approve")
|
||||
if _, err := askABuild(warranted, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", "main"); !errors.As(err, &policy) {
|
||||
t.Fatalf("1146/AC9: the warrant covered a path the merge does not hold: %v", err)
|
||||
}
|
||||
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queue, err := link.ReadQueue(ctx, js, link.TheBuildMachine)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(queue.Asks) != 1 {
|
||||
t.Fatalf("1146/AC9: the build queue holds %+v, want the one first build", queue)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1146/AC9: the warrant the controller checked covers exactly the repository on the git seat and the module paths the
|
||||
// merge holds; nothing else passes verbMayAsk on it.
|
||||
func TestTheFirstBuildWarrantCoversOnlyItsRepositoryAndItsModulePaths(t *testing.T) {
|
||||
ctx := withFirstBuildWarrant(t.Context(), "novox/mesh-delivery", []string{"", "view"}, "the Approve")
|
||||
w := firstBuildWarrantIn(ctx)
|
||||
for _, c := range []struct {
|
||||
source buildSource
|
||||
path string
|
||||
covers bool
|
||||
}{
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", true},
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "/view/", true},
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", false},
|
||||
{buildSource{Repository: "agent/mesh-delivery", Seat: "git"}, "", false},
|
||||
{buildSource{Repository: "novox/mesh-delivery"}, "", false},
|
||||
{buildSource{Repository: "novox/mesh-delivery", Seat: "elsewhere"}, "", false},
|
||||
} {
|
||||
if got := w.covers(c.source, c.path); got != c.covers {
|
||||
t.Errorf("1146/AC9: %v at %q covered = %v, want %v", c.source, c.path, got, c.covers)
|
||||
}
|
||||
}
|
||||
if firstBuildWarrantIn(t.Context()).covers(buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "") {
|
||||
t.Error("1146/AC9: a context with no warrant covers an ask")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// firstBuilds asks the first build of every module a merge adds to a repository no registered module is built from, when
|
||||
// the merge was announced as made on the operator's Approve of that head (novox/hq ADR 0432, issue 1146): the modules'
|
||||
// directories are the ones the announcement says hold a module.json at the merge commit, as touchedBy reads them. The
|
||||
// merge is kept, so the build's outcome is judged by the rule that registers it (mayRegisterFrom, approvedMove).
|
||||
//
|
||||
// **Complete in either order, with no timer** (ADR 0426): when mesh-delivery's word is kept first, it is found here; when
|
||||
// the forge's merge is heard first, the merge is kept as a first merge waiting for the word, and keepingDeliveryMerges
|
||||
// runs this again when the word is kept (resumeFirstBuilds). A word that says no (a Decline, another head) asks nothing and
|
||||
// says why. Answers whether the merge was dealt with here.
|
||||
func (f following) firstBuilds(ctx context.Context, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository, now time.Time) (bool, error) {
|
||||
inv := f.open.inventory
|
||||
repository := m.Owner + "/" + m.Repo
|
||||
added, why, err := firstBuildsOf(ctx, inv, m, entries, read)
|
||||
switch {
|
||||
case errors.Is(err, errFirstNotApproved):
|
||||
fmt.Printf("%s merged into %s (%.8s) adds %s, and no build is asked: %v\n", repository, m.Base, m.Commit,
|
||||
strings.Join(addedDirs(m, entries, read), ", "), err)
|
||||
return true, nil
|
||||
case errors.Is(err, errNotAnnounced):
|
||||
// Kept, waiting for mesh-delivery's word: a restart keeps it, and the word's arrival asks the builds.
|
||||
if kerr := keepFirstMerge(ctx, inv, m, now, true); kerr != nil {
|
||||
return true, notNow(kerr)
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s) adds %s; it is kept, and the first build is asked when mesh-delivery "+
|
||||
"announces the merge on the operator's Approve\n", repository, m.Base, m.Commit,
|
||||
strings.Join(addedDirs(m, entries, read), ", "))
|
||||
return true, nil
|
||||
case err != nil:
|
||||
return true, notNow(err)
|
||||
case len(added) == 0:
|
||||
return false, nil
|
||||
}
|
||||
if err := askFirstBuilds(ctx, inv, m, now, added, why); err != nil {
|
||||
return true, notNow(err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// keepFirstMerge keeps a first merge as the controller heard it, waiting (no plan answers it).
|
||||
func keepFirstMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, waitingForWord bool) error {
|
||||
merged, err := time.Parse(time.RFC3339Nano, m.MergedAt)
|
||||
if err != nil {
|
||||
merged = now
|
||||
}
|
||||
event, err := json.Marshal(keptMerge{SourceMoved: m, FirstBuild: waitingForWord})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = inv.AddMerge(ctx, inventory.BatchedMerge{Repository: m.Owner + "/" + m.Repo, Branch: m.Base, Commit: m.Commit,
|
||||
Merged: merged.UTC(), Heard: now, Event: event})
|
||||
return err
|
||||
}
|
||||
|
||||
// askFirstBuilds keeps the merge, asks the builds, and marks the merge as no longer waiting for them.
|
||||
func askFirstBuilds(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, added []string,
|
||||
why string) error {
|
||||
repository := m.Owner + "/" + m.Repo
|
||||
if err := keepFirstMerge(ctx, inv, m, now, false); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so the "+
|
||||
"first build of %s is asked\n", repository, m.Base, m.Commit, why, strings.Join(added, ", "))
|
||||
// The ask carries the warrant just checked, for this repository and these module paths alone: the guard of a build
|
||||
// through a verb (verbMayAsk) lets exactly that through.
|
||||
paths := make([]string, 0, len(added))
|
||||
for _, dir := range added {
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
paths = append(paths, dir)
|
||||
}
|
||||
askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, m, buildSource{Repository: repository, Seat: gitSeat}, added)
|
||||
// Answered: it no longer waits, so the word's arrival again asks nothing twice.
|
||||
return inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false)
|
||||
}
|
||||
|
||||
// resumeFirstBuilds runs firstBuilds for the first merges waiting for the word mesh-delivery just announced for a pull
|
||||
// request's head: the other order of the same two events.
|
||||
func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error {
|
||||
actingOnMerges.Lock()
|
||||
defer actingOnMerges.Unlock()
|
||||
inv := f.open.inventory
|
||||
waiting, err := inv.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if !k.FirstBuild || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number ||
|
||||
k.HeadSHA != d.HeadSHA {
|
||||
continue
|
||||
}
|
||||
entries, err := inv.Catalogued(ctx)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
read, err := readForPlanning(ctx, inv)
|
||||
if err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
added, why, err := firstBuildsOf(ctx, inv, k.SourceMoved, entries, read)
|
||||
switch {
|
||||
case errors.Is(err, errFirstNotApproved):
|
||||
fmt.Printf("%s/%s merged into %s (%.8s) was waiting for mesh-delivery's word, and no build is asked: %v\n",
|
||||
k.Owner, k.Repo, k.Base, k.Commit, err)
|
||||
if aerr := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); aerr != nil {
|
||||
return notNow(aerr)
|
||||
}
|
||||
case err != nil:
|
||||
return notNow(err)
|
||||
case len(added) > 0:
|
||||
if err := askFirstBuilds(ctx, inv, k.SourceMoved, time.Now().UTC(), added, why); err != nil {
|
||||
return notNow(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errFirstNotApproved says a merge into a repository the mesh builds nothing from is not one the operator approved.
|
||||
var errFirstNotApproved = errors.New("it is no merge made on the operator's Approve of its head, so a hand `build` " +
|
||||
"registers a module from a repository the mesh does not build yet")
|
||||
|
||||
// addedDirs are the module directories the merge adds to a repository no registered module is built from, none where
|
||||
// the merge is not into the trunk or some module is built from the repository already.
|
||||
func addedDirs(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository) []string {
|
||||
if m.Base != "main" {
|
||||
return nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
if !e.Provided && sameRepository(e.Source.Repository, m) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
_, added, _ := touchedBy(nil, entries, m, read)
|
||||
return added
|
||||
}
|
||||
|
||||
// firstBuildsOf is the directories whose first build a merge asks, and the warrant it was made on; none when the merge
|
||||
// is no first merge into a repository the mesh builds nothing from. errFirstNotApproved when it is and the word on record
|
||||
// is no Approve of its head; errNotAnnounced when mesh-delivery's word is not kept yet.
|
||||
func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved, entries []inventory.Entry,
|
||||
read map[string][]inventory.ReadRepository) ([]string, string, error) {
|
||||
added := addedDirs(m, entries, read)
|
||||
if len(added) == 0 {
|
||||
return nil, "", nil
|
||||
}
|
||||
why, err := approvedAnnounced(ctx, inv, m.Owner, m.Repo, keptMerge{SourceMoved: m})
|
||||
switch {
|
||||
case err == nil:
|
||||
return added, why, nil
|
||||
case errors.Is(err, errWordNotAnApprove):
|
||||
return nil, "", fmt.Errorf("%w (%v)", errFirstNotApproved, err)
|
||||
}
|
||||
// Not announced yet, or the bus or the store could not be asked: the caller waits or tries again, and asks nothing.
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word.
|
||||
const probeFirstBuildsID = "D18"
|
||||
|
||||
// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises.
|
||||
const kindFirstBuildUnasked = "first-build-unasked"
|
||||
|
||||
// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before
|
||||
// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds
|
||||
// at once, and the merge is said, never dropped, once it is past this (issue 1146).
|
||||
const firstBuildWaitBound = 10 * time.Minute
|
||||
|
||||
// firstBuildObservations are the conditions of the first merges waiting past the bound, one for each; a merge answered is
|
||||
// no longer listed as waiting, so its condition clears with the next look.
|
||||
func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []conditions.Observation {
|
||||
var out []conditions.Observation
|
||||
for _, w := range waiting {
|
||||
k := announcedOf(w)
|
||||
if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound {
|
||||
continue
|
||||
}
|
||||
repository := k.Owner + "/" + k.Repo
|
||||
for_ := now.Sub(w.Heard).Round(time.Minute)
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeMerge,
|
||||
ID: fmt.Sprintf("%s_%.8s", repository, k.Commit), Kind: kindFirstBuildUnasked, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the merge %.8s of %s adds modules the mesh builds nothing from yet, and has waited %s for "+
|
||||
"mesh-delivery's announcement of the operator's Approve of its head: no build is asked", k.Commit, repository, for_),
|
||||
Said: fmt.Sprintf("%s at %s waits for mesh-delivery's word on pull request %d (head %.8s)", repository, k.Commit,
|
||||
k.Number, k.HeadSHA),
|
||||
Headline: fmt.Sprintf("The first build of %s is not asked: no Approve was announced", repository),
|
||||
Explanation: fmt.Sprintf("The merge %.8s into %s adds modules the mesh builds nothing from yet. It registers them "+
|
||||
"only when mesh-delivery announces that it merged the pull request on your Approve of that head, and no such "+
|
||||
"announcement has come in %s. The merge is kept, and the builds are asked the moment the announcement is kept.",
|
||||
k.Commit, repository, for_),
|
||||
Resolved: "mesh-delivery announces the merge, or a hand `build` registers the modules",
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid.
|
||||
func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
waiting, err := d.open.inventory.WaitingMerges(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return firstBuildObservations(waiting, time.Now().UTC()), nil
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1146/AC7, whole against a real bus and the store, in both orders of the two events (novox/hq ADR 0432, issue 1146):
|
||||
// mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's merge,
|
||||
// heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the controller
|
||||
// consumes with the router's record of the ask in its bucket, come in either order; each way the first build of the
|
||||
// module the merge commit holds is asked exactly once, its outcome registers the module, and it is in the catalogue.
|
||||
// Without the Approve in the record nothing is asked. Store-backed: skips without one.
|
||||
func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegistersTheModule(t *testing.T) {
|
||||
const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000"
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
wordFirst bool
|
||||
approved bool // the router's record holds the operator's Approve of the merged head
|
||||
wantRegiter bool
|
||||
}{
|
||||
{what: "the forge's merge before mesh-delivery's word", approved: true, wantRegiter: true},
|
||||
{what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegiter: true},
|
||||
{what: "a word whose record holds no Approve", wordFirst: true},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
// The real ask, guard and queue included: only its answers are kept, so the build's outcome can be matched.
|
||||
aForgeAndABuildSeat(t, open)
|
||||
// Written by the controller's merge path on its own goroutine, read by the test: under one lock.
|
||||
var mu sync.Mutex
|
||||
var asks [][3]string
|
||||
var ids []string
|
||||
asked := func() [][3]string { mu.Lock(); defer mu.Unlock(); return slices.Clone(asks) }
|
||||
realAsk := askABuild
|
||||
askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) {
|
||||
id, err := realAsk(c, source, path, ref)
|
||||
if err == nil {
|
||||
mu.Lock()
|
||||
asks = append(asks, [3]string{source.Repository, path, ref})
|
||||
ids = append(ids, id)
|
||||
mu.Unlock()
|
||||
}
|
||||
return id, err
|
||||
}
|
||||
t.Cleanup(func() { askABuild = realAsk })
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
|
||||
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
if c.approved {
|
||||
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
following := following{open: open}
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims, kept: following.resumeFirstBuilds}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.AssertServingConsumers(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serving, stop := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = server.Serve(serving) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
|
||||
word := func() {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(a.merge)
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the word being kept", func() bool {
|
||||
_, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, head)
|
||||
return err == nil && kept
|
||||
})
|
||||
time.Sleep(500 * time.Millisecond) // the keeper's hook runs as the word is kept
|
||||
}
|
||||
forge := func() {
|
||||
t.Helper()
|
||||
m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7,
|
||||
MergedAt: time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"},
|
||||
ModuleDirs: []string{"."}, ModuleDirsSaid: true}
|
||||
if err := following.SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if c.wordFirst {
|
||||
word()
|
||||
forge()
|
||||
} else {
|
||||
forge()
|
||||
if len(asked()) != 0 {
|
||||
t.Fatalf("1146/AC7: the merge asked %v before mesh-delivery's word was kept", asked())
|
||||
}
|
||||
word()
|
||||
}
|
||||
if !c.wantRegiter {
|
||||
if len(asked()) != 0 {
|
||||
t.Fatalf("1146/AC7: a merge whose record holds no Approve asked %v", asked())
|
||||
}
|
||||
return
|
||||
}
|
||||
want := [3]string{"novox/mesh-delivery", "", "main"}
|
||||
becomes(t, 10*time.Second, "the first build asked", func() bool { return len(asked()) >= 1 })
|
||||
if got := asked(); len(got) != 1 || got[0] != want {
|
||||
t.Fatalf("1146/AC7: the merge asked %v, want %v once", got, want)
|
||||
}
|
||||
mu.Lock()
|
||||
first := ids[0]
|
||||
mu.Unlock()
|
||||
build := onTrunk(first, "novox/mesh-delivery", "git", "", map[string]any{"module": "mesh-delivery-view", "version": "1"})
|
||||
build.Commit = commit
|
||||
if _, _, err := takeIn(ctx, inv, build); err != nil {
|
||||
t.Fatalf("1146/AC7: the outcome of the first build was refused: %v", err)
|
||||
}
|
||||
src, err := inv.SourceOf(ctx, "mesh-delivery-view")
|
||||
if err != nil || src.Repository != "novox/mesh-delivery" {
|
||||
t.Fatalf("1146/AC7: the module is built from %q (%v)", src.Repository, err)
|
||||
}
|
||||
if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); !known {
|
||||
t.Fatal("1146/AC7: the module is not in the catalogue")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) {
|
||||
t.Helper()
|
||||
entries, err := inv.Catalogued(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == module {
|
||||
return e, true
|
||||
}
|
||||
}
|
||||
return inventory.Entry{}, false
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1146/AC6: the first merge into a repository no registered module is built from asks the first build of every module
|
||||
// the merge commit holds when it was made on the operator's Approve of that head, and asks nothing without it.
|
||||
func TestAFirstMergeIntoANewRepositoryAsksItsModulesBuildsOnlyOnTheApprove(t *testing.T) {
|
||||
const head = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||
now := time.Date(2026, 10, 19, 0, 6, 0, 0, time.UTC)
|
||||
mergedAt := now.Add(-time.Hour).Format(time.RFC3339Nano)
|
||||
heard := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: "0177ea7a",
|
||||
Number: 1, MergedAt: mergedAt, ModuleDirsSaid: true, ModuleDirs: []string{"watchman", "view"},
|
||||
Paths: []string{"view/module.json", "watchman/module.json", "README.md"}}
|
||||
built := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git"}}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
change func(*anApprovedMerge, *link.SourceMoved, *[]inventory.Entry)
|
||||
unkept bool
|
||||
dirs []string
|
||||
refused bool
|
||||
waits bool
|
||||
}{
|
||||
{what: "the Approve of the merged head", dirs: []string{"view", "watchman"}},
|
||||
{what: "a Decline", refused: true, change: func(a *anApprovedMerge, _ *link.SourceMoved, _ *[]inventory.Entry) {
|
||||
a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline"
|
||||
}},
|
||||
{what: "no word of mesh-delivery yet, however long ago the merge was made", unkept: true, waits: true},
|
||||
{what: "a module already built from the repository", change: func(_ *anApprovedMerge, _ *link.SourceMoved, e *[]inventory.Entry) {
|
||||
*e = append(*e, inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}})
|
||||
}},
|
||||
{what: "a merge into another branch", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) { m.Base = "feat" }},
|
||||
{what: "a merge that holds no module.json", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) {
|
||||
m.ModuleDirs, m.Paths = nil, []string{"README.md"}
|
||||
}},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
a.merge.Number = 1
|
||||
m, entries := heard, []inventory.Entry{built}
|
||||
if c.change != nil {
|
||||
c.change(&a, &m, &entries)
|
||||
}
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
dirs, _, err := firstBuildsOf(t.Context(), nil, m, entries, nil)
|
||||
switch {
|
||||
case c.waits && !errors.Is(err, errNotAnnounced):
|
||||
t.Fatalf("1146/AC6: %s: want a wait for the word, got %v", c.what, err)
|
||||
case c.refused && !errors.Is(err, errFirstNotApproved):
|
||||
t.Fatalf("1146/AC6: %s: want the Approve missing, got %v (dirs %v)", c.what, err, dirs)
|
||||
case !c.refused && !c.waits && err != nil:
|
||||
t.Fatalf("1146/AC6: %s: %v", c.what, err)
|
||||
case !slices.Equal(dirs, c.dirs):
|
||||
t.Fatalf("1146/AC6: %s: first builds asked of %v, want %v", c.what, dirs, c.dirs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 1146/AC8: a first merge waiting past the bound for mesh-delivery's word is said, naming the repository, the commit and
|
||||
// the missing word; one inside the bound, one that is no first merge, and one answered (no longer waiting) are not.
|
||||
func TestAFirstMergeWaitingPastItsBoundIsSaidAndClearsWhenAnswered(t *testing.T) {
|
||||
now := time.Date(2026, 10, 19, 1, 0, 0, 0, time.UTC)
|
||||
waiting := func(commit string, first bool, heard time.Time) inventory.BatchedMerge {
|
||||
event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main",
|
||||
Commit: commit, Number: 1, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: first})
|
||||
return inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit, Heard: heard, Event: event}
|
||||
}
|
||||
got := firstBuildObservations([]inventory.BatchedMerge{
|
||||
waiting("0177ea7a00", true, now.Add(-firstBuildWaitBound-time.Minute)),
|
||||
waiting("1111111100", true, now.Add(-time.Minute)),
|
||||
waiting("2222222200", false, now.Add(-time.Hour)),
|
||||
}, now)
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("1146/AC8: %d conditions raised, want the one merge past the bound: %+v", len(got), got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Kind != kindFirstBuildUnasked || o.Scope != conditions.ScopeMerge || o.Severity != conditions.Warning ||
|
||||
!strings.Contains(o.Summary, "novox/mesh-delivery") || !strings.Contains(o.Summary, "0177ea7a") ||
|
||||
!strings.Contains(o.Summary, "announcement of the operator's Approve") {
|
||||
t.Fatalf("1146/AC8: the condition is %+v", o)
|
||||
}
|
||||
// Answered, the merge is no longer listed as waiting: nothing is observed, and the condition clears.
|
||||
if cleared := firstBuildObservations(nil, now); len(cleared) != 0 {
|
||||
t.Fatalf("1146/AC8: an answered merge still raises %+v", cleared)
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,7 @@ func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
||||
resolver := entry("dnsmasq", "modules/dnsmasq")
|
||||
resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}}
|
||||
delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery")
|
||||
delivery.Manifest.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||
webApp := entry("photos", "modules/photos")
|
||||
entries := []inventory.Entry{gitea, resolver, delivery, webApp}
|
||||
// photos stands on gitea: a change to gitea moves photos as its dependent.
|
||||
@@ -44,7 +45,7 @@ func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) {
|
||||
{"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
{"nothing any module's build reads", []string{"README.md"}, "none"},
|
||||
{"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf},
|
||||
{"a new module on the controller's own path, by its own class", []string{"next/mesh-delivery/module.json"}, inventory.ClassCore},
|
||||
{"a new module on the controller's own path, by its own class", []string{"next/mesh-host/module.json"}, inventory.ClassCore},
|
||||
{"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore},
|
||||
} {
|
||||
if got := plan(c.paths...).Class; got != c.want {
|
||||
|
||||
@@ -69,7 +69,7 @@ var (
|
||||
// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's
|
||||
// delivery budget; how many readings its gate takes is gateRuleOf's.
|
||||
func coreModule(module string, m catalogue.Manifest) bool {
|
||||
if _, own := onTheControllersPath[module]; own {
|
||||
if onTheOwnPath(module, m) {
|
||||
return true
|
||||
}
|
||||
for _, c := range m.Claims {
|
||||
@@ -106,7 +106,7 @@ func gateRuleOf(module string, m catalogue.Manifest) gateRule {
|
||||
case coreComponent(module) == lease.ComponentNodeTools:
|
||||
return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " +
|
||||
"after the node-engine's witness window (novox/hq ADR 0351)"}
|
||||
case module == catalogue.DeliverySeat:
|
||||
case m.ClaimsSeat(catalogue.DeliverySeat):
|
||||
return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"}
|
||||
case coreModule(module, m):
|
||||
return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
|
||||
}{
|
||||
{"mesh-controller", nil, 1, 0, true, "the controller's rule"},
|
||||
{"node-tools", nil, 1, 0, true, "the tool runner's rule"},
|
||||
{catalogue.DeliverySeat, nil, 1, 0, false, "mesh-delivery's rule"},
|
||||
{catalogue.DeliverySeat, []catalogue.Claim{{Name: catalogue.DeliverySeat}}, 1, 0, false, "mesh-delivery's rule"},
|
||||
{"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
{"nats", nil, 3, 2 * time.Minute, false, "the core's rule"},
|
||||
@@ -56,7 +56,12 @@ func TestEachModuleHasItsGateRule(t *testing.T) {
|
||||
// Every module that was core stays core for its delivery budget and its walk: only its readings change.
|
||||
func TestTheCoreStaysTheCore(t *testing.T) {
|
||||
for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} {
|
||||
if !coreModule(m, catalogue.Manifest{Module: m}) {
|
||||
// The delivery's owner is core by the claim of the seat, not by its name (novox/hq ADR 0437 §3, issue 1210).
|
||||
claims := []catalogue.Claim(nil)
|
||||
if m == catalogue.DeliverySeat {
|
||||
claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||
}
|
||||
if !coreModule(m, catalogue.Manifest{Module: m, Claims: claims}) {
|
||||
t.Errorf("%s is no longer judged as the core for its class", m)
|
||||
}
|
||||
}
|
||||
@@ -78,7 +83,11 @@ func aGateOn(module, node string, since time.Time) *gateOn {
|
||||
}
|
||||
|
||||
func (o *gateOn) read(at time.Time, h health, why string) {
|
||||
r := gateRuleOf(o.module, catalogue.Manifest{Module: o.module})
|
||||
m := catalogue.Manifest{Module: o.module}
|
||||
if o.module == catalogue.DeliverySeat { // today's module, which claims the seat
|
||||
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}}
|
||||
}
|
||||
r := gateRuleOf(o.module, m)
|
||||
floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder)
|
||||
tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h},
|
||||
worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at)
|
||||
|
||||
@@ -77,15 +77,11 @@ func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationHeartbeatGap, Subject: "anchor", Took: time.Minute})
|
||||
got := summarise(ds)
|
||||
if len(got) != 2 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
if len(got) != 1 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if !strings.Contains(got[1].Suggests, "3m0s") {
|
||||
t.Fatalf("a minute between words suggests %q", got[1].Suggests)
|
||||
}
|
||||
}
|
||||
|
||||
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// Last heard (novox/hq ADR 0433 §3, issue 1183): one answer to when the controller last heard a machine, read from the
|
||||
// watchman's presence. The alive message that used to keep last_seen fresh is gone; a report, which is rare, is the
|
||||
// answer only where presence has no key for the machine.
|
||||
|
||||
// presenceReader reads presence.<node>'s entry for each node; a node without an entry is left out.
|
||||
type presenceReader func(ctx context.Context, nodes []string) (map[string]presenceEntry, error)
|
||||
|
||||
// lastHeard is when a machine was last heard: now when presence says it is online, the entry's time when it says offline
|
||||
// (the time it went), and the machine's last report only when presence has no entry for it. Zero is never heard.
|
||||
func lastHeard(report time.Time, entry *presenceEntry, now time.Time) time.Time {
|
||||
switch {
|
||||
case entry == nil:
|
||||
return report
|
||||
case entry.Online:
|
||||
return now
|
||||
default:
|
||||
return entry.At
|
||||
}
|
||||
}
|
||||
|
||||
// nodesHeard returns the nodes with their last-heard time (LastSeen) as lastHeard says it. Unreadable presence is
|
||||
// returned as an error with the nodes unchanged, so a caller says it and shows the reports.
|
||||
func nodesHeard(ctx context.Context, nodes []inventory.Node, read presenceReader, now time.Time) ([]inventory.Node, error) {
|
||||
names := make([]string, len(nodes))
|
||||
for i, n := range nodes {
|
||||
names[i] = n.Name
|
||||
}
|
||||
entries, err := read(ctx, names)
|
||||
if err != nil {
|
||||
return nodes, fmt.Errorf("what the watchman says of the machines' presence cannot be read: %w", err)
|
||||
}
|
||||
out := make([]inventory.Node, len(nodes))
|
||||
for i, n := range nodes {
|
||||
var entry *presenceEntry
|
||||
if e, ok := entries[n.Name]; ok {
|
||||
entry = &e
|
||||
}
|
||||
n.LastSeen = lastHeard(n.LastSeen, entry, now)
|
||||
out[i] = n
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// heardNodes is every node, read with its last-heard time from the watchman's presence over a connection of its own. A bus
|
||||
// that cannot be reached is said on standard error and the reports stand, which reads a machine as quiet rather than well.
|
||||
func (h *stores) heardNodes(ctx context.Context) ([]inventory.Node, error) {
|
||||
nodes, err := h.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return h.withPresence(ctx, nodes), nil
|
||||
}
|
||||
|
||||
// heardNode is one node, read the same way.
|
||||
func (h *stores) heardNode(ctx context.Context, name string) (inventory.Node, error) {
|
||||
n, err := h.inventory.NodeByName(ctx, name)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
return h.withPresence(ctx, []inventory.Node{n})[0], nil
|
||||
}
|
||||
|
||||
func (h *stores) withPresence(ctx context.Context, nodes []inventory.Node) []inventory.Node {
|
||||
address, err := broker.BusAddress()
|
||||
var js *broker.JetStream
|
||||
if err == nil {
|
||||
js, err = broker.Dial(address)
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "what the watchman says of the machines' presence cannot be read: %v\n", err)
|
||||
return nodes
|
||||
}
|
||||
defer js.Close()
|
||||
read := func(ctx context.Context, names []string) (map[string]presenceEntry, error) {
|
||||
return readPresence(ctx, js.Conn(), names)
|
||||
}
|
||||
heard, err := nodesHeard(ctx, nodes, read, time.Now())
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
}
|
||||
return heard
|
||||
}
|
||||
|
||||
// heardFromAt says when a node was last heard from, in a form somebody can act on.
|
||||
//
|
||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never spoken did not finish
|
||||
// joining; a node last heard from an hour ago is running an hour-old picture of the mesh.
|
||||
func heardFromAt(n inventory.Node, now time.Time) string {
|
||||
if n.LastSeen.IsZero() {
|
||||
return "never spoken"
|
||||
}
|
||||
if silent := now.Sub(n.LastSeen); silent > SilentFor {
|
||||
return "out of touch " + roughly(silent)
|
||||
}
|
||||
return "here"
|
||||
}
|
||||
|
||||
// notHeardLately is the nodes never heard from or not within the hour: different from failing, for a machine that says
|
||||
// nothing may be new, switched off or unreachable, and none of those is a machine that tried and could not.
|
||||
func notHeardLately(nodes []inventory.Node, now time.Time) []inventory.Node {
|
||||
var out []inventory.Node
|
||||
for _, n := range nodes {
|
||||
if n.LastSeen.IsZero() || now.Sub(n.LastSeen) > time.Hour {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1183/AC5: with the watchman's presence bucket holding a node online and no alive message ever sent, the node reads here
|
||||
// in `nodes` four minutes later by the controller's clock, however old its last report; and an alive message still sent
|
||||
// by a node-engine is published to the same bus without error and changes nothing.
|
||||
func TestAnOnlineNodeReadsHereWithNoAliveMessage(t *testing.T) { // 1183/AC5
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if _, err := broker.RaiseBuckets(js, []broker.Bucket{
|
||||
{Module: sdkdelivery.WatchmanModule, Name: "presence", History: sdkdelivery.PresenceHistory},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.PresenceBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := func(node string, online bool) {
|
||||
t.Helper()
|
||||
key, err := sdkdelivery.PresenceNodeKey(node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := sdkdelivery.EncodePresence(sdkdelivery.Presence{Online: online})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, key, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put("ace", true)
|
||||
put("laptop", false)
|
||||
// A node-engine still sending its alive message: harmless.
|
||||
if err := js.Conn().Publish("mesh.control.ace.alive", []byte(`{"node":"ace"}`)); err != nil {
|
||||
t.Fatalf("an alive message could not be published: %v", err)
|
||||
}
|
||||
if err := js.Conn().Flush(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
later := time.Now().Add(4 * time.Minute)
|
||||
nodes := []inventory.Node{{Name: "ace", LastSeen: time.Now().Add(-time.Hour)},
|
||||
{Name: "laptop", LastSeen: time.Now().Add(-time.Hour)}}
|
||||
read := func(ctx context.Context, names []string) (map[string]presenceEntry, error) {
|
||||
return readPresence(ctx, js.Conn(), names)
|
||||
}
|
||||
heard, err := nodesHeard(ctx, nodes, read, later)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := heardFromAt(heard[0], later); got != "here" {
|
||||
t.Fatalf("a node online by presence reads %q four minutes later", got)
|
||||
}
|
||||
if got := heardFromAt(heard[1], later); got == "here" {
|
||||
t.Fatalf("a node offline by presence reads %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// aPresenceFake is the watchman's presence bucket as a reader sees it.
|
||||
func aPresenceFake(entries map[string]presenceEntry, err error) presenceReader {
|
||||
return func(_ context.Context, _ []string) (map[string]presenceEntry, error) { return entries, err }
|
||||
}
|
||||
|
||||
// 1183/AC1: one function says when a machine was last heard: online is now, offline is the entry's time, and only where
|
||||
// presence has no key, the last report.
|
||||
func TestLastHeardIsPresenceThenTheReport(t *testing.T) { // 1183/AC1
|
||||
now := time.Now()
|
||||
report := now.Add(-time.Hour)
|
||||
if got := lastHeard(report, nil, now); !got.Equal(report) {
|
||||
t.Fatalf("no presence key: last heard is %v, want the report %v", got, report)
|
||||
}
|
||||
if got := lastHeard(report, &presenceEntry{Online: true, At: now.Add(-9 * time.Hour)}, now); !got.Equal(now) {
|
||||
t.Fatalf("online: last heard is %v, want now", got)
|
||||
}
|
||||
offlineAt := now.Add(-4 * time.Minute)
|
||||
if got := lastHeard(now.Add(-time.Minute), &presenceEntry{At: offlineAt}, now); !got.Equal(offlineAt) {
|
||||
t.Fatalf("offline: last heard is %v, want the entry's time %v, not a report's", got, offlineAt)
|
||||
}
|
||||
if got := lastHeard(time.Time{}, nil, now); !got.IsZero() {
|
||||
t.Fatalf("never heard and no key: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The machines facts, the nodes list, node and status all take that one function.
|
||||
func TestEveryReaderOfLastHeardTakesPresence(t *testing.T) { // 1183/AC1
|
||||
now := time.Now()
|
||||
m := machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute)}
|
||||
applyPresence(&m, nil, &presenceEntry{At: now.Add(-4 * time.Minute)}, now)
|
||||
if !m.lastHeard.Equal(now.Add(-4 * time.Minute)) {
|
||||
t.Fatalf("machine facts take a report over the offline entry: %v", m.lastHeard)
|
||||
}
|
||||
}
|
||||
|
||||
func twoNodes(now time.Time) []inventory.Node {
|
||||
return []inventory.Node{
|
||||
{Name: "anchor", LastSeen: now.Add(-time.Hour)},
|
||||
{Name: "laptop", LastSeen: now.Add(-30 * time.Second)},
|
||||
{Name: "fresh"},
|
||||
}
|
||||
}
|
||||
|
||||
// 1183/AC2: a machine online by presence whose last report is an hour old reads here, is not quiet and is not silent; one
|
||||
// offline for 4 minutes reads out of touch.
|
||||
func TestOnlineByPresenceReadsHereWhateverTheReport(t *testing.T) { // 1183/AC2
|
||||
now := time.Now()
|
||||
read := aPresenceFake(map[string]presenceEntry{
|
||||
"anchor": {Online: true, At: now.Add(-6 * time.Hour)},
|
||||
"laptop": {Online: false, At: now.Add(-4 * time.Minute)},
|
||||
}, nil)
|
||||
nodes, err := nodesHeard(context.Background(), twoNodes(now), read, now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := heardFromAt(nodes[0], now); got != "here" {
|
||||
t.Errorf("anchor online with an hour-old report reads %q", got)
|
||||
}
|
||||
if got := heardFromAt(nodes[1], now); got != "out of touch 4m" {
|
||||
t.Errorf("laptop offline 4 minutes reads %q", got)
|
||||
}
|
||||
if got := heardFromAt(nodes[2], now); got != "never spoken" {
|
||||
t.Errorf("a machine with no key and no report reads %q", got)
|
||||
}
|
||||
quiet := notHeardLately(nodes, now)
|
||||
if len(quiet) != 1 || quiet[0].Name != "fresh" {
|
||||
t.Errorf("quiet is %+v: an online machine is not quiet, and a machine offline 4 minutes is not quiet yet", quiet)
|
||||
}
|
||||
f := &machineFacts{name: "anchor", lastHeard: nodes[0].LastSeen}
|
||||
if f.silent(now) {
|
||||
t.Errorf("a machine online by presence is silent")
|
||||
}
|
||||
}
|
||||
|
||||
// Unreadable presence is said, and the report stands as it was.
|
||||
func TestUnreadablePresenceLeavesTheReport(t *testing.T) { // 1183/AC1
|
||||
now := time.Now()
|
||||
nodes, err := nodesHeard(context.Background(), twoNodes(now), aPresenceFake(nil, errors.New("no bus")), now)
|
||||
if err == nil || !strings.Contains(err.Error(), "no bus") {
|
||||
t.Fatalf("an unreadable presence is not said: %v", err)
|
||||
}
|
||||
if !nodes[0].LastSeen.Equal(now.Add(-time.Hour)) {
|
||||
t.Fatalf("the report was replaced: %v", nodes[0].LastSeen)
|
||||
}
|
||||
}
|
||||
|
||||
// 1183/AC4: where the watchman gives no verdict a machine is silent after a fixed three minutes, whatever interval a
|
||||
// node-engine once said.
|
||||
func TestTheSilentBoundIsFixed(t *testing.T) { // 1183/AC4
|
||||
now := time.Now()
|
||||
for _, c := range []struct {
|
||||
ago time.Duration
|
||||
want bool
|
||||
}{{179 * time.Second, false}, {181 * time.Second, true}} {
|
||||
m := machineFacts{name: "laptop", lastHeard: now.Add(-c.ago)}
|
||||
if got := m.silent(now); got != c.want {
|
||||
t.Errorf("heard %v ago: silent %v, want %v", c.ago, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// aLongGroup is members as JSON over 200 KB: 1666 paths of the shape of mesh-delivery #5's change (novox/hq issue 1272).
|
||||
func aLongGroup(t *testing.T) string {
|
||||
t.Helper()
|
||||
var paths []string
|
||||
for i := 0; i < 1666; i++ {
|
||||
paths = append(paths, fmt.Sprintf("modules/mesh-delivery-forge-bridge/vendor/github.com/nats-io/nats-server/v2/server/certstore/certstore_windows_file_%04d.go", i))
|
||||
}
|
||||
body, err := json.Marshal([]map[string]any{{"id": "novox/mesh-delivery#5", "paths": paths}, {"id": "novox/mesh-catalog#374"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(body) < 200<<10 {
|
||||
t.Fatalf("the members are only %d bytes", len(body))
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
|
||||
func readBackFromTheCommand(t *testing.T, argv []string, members string) {
|
||||
t.Helper()
|
||||
for _, a := range argv {
|
||||
if len(a) > 64<<10 {
|
||||
t.Errorf("an argument of %d bytes is handed to the command", len(a))
|
||||
}
|
||||
}
|
||||
i := slices.Index(argv, "--members-file")
|
||||
if i < 0 {
|
||||
t.Fatalf("the command does not name a members file: %.200v", argv)
|
||||
}
|
||||
raw, err := os.ReadFile(argv[i+1])
|
||||
if err != nil || string(raw) != members {
|
||||
t.Errorf("the file does not hold the whole members list (%d bytes, %v)", len(raw), err)
|
||||
}
|
||||
removeMembersFile(argv)
|
||||
if _, err := os.Stat(argv[i+1]); !os.IsNotExist(err) {
|
||||
t.Errorf("the file is still there after the command: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeliveryCheckHandsALongMembersListInAFile(t *testing.T) { // 1272/AC1
|
||||
members := aLongGroup(t)
|
||||
argv, err := argvFor("delivery-check", map[string]any{"group": "feat/x", "members": members})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if argv[0] != "delivery" || argv[1] != "check" || !slices.Contains(argv, "--group") {
|
||||
t.Errorf("the command is %.100v", strings.Join(argv, " "))
|
||||
}
|
||||
readBackFromTheCommand(t, argv, members)
|
||||
}
|
||||
|
||||
func TestDeliveryOrderHandsALongMembersListInAFile(t *testing.T) { // 1272/AC2
|
||||
members := aLongGroup(t)
|
||||
argv, err := argvFor("delivery-order", map[string]any{"members": members})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if argv[0] != "delivery" || argv[1] != "order" {
|
||||
t.Errorf("the command is %.100v", strings.Join(argv, " "))
|
||||
}
|
||||
readBackFromTheCommand(t, argv, members)
|
||||
}
|
||||
|
||||
func TestAShortMembersListIsStillTheArgumentItWas(t *testing.T) { // 1272/AC3
|
||||
members := `[{"id":"novox/a#1"},{"id":"novox/b#2"}]`
|
||||
argv, err := argvFor("delivery-check", map[string]any{"group": "feat/x", "members": members, "ready": "true"})
|
||||
want := []string{"delivery", "check", "--group", "feat/x", "--members", members, "--ready"}
|
||||
if err != nil || !slices.Equal(argv, want) {
|
||||
t.Errorf("a short list gives %v, %v, want %v", argv, err, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheCommandReadsTheMembersFromTheFileAsFromTheArgument(t *testing.T) { // 1272/AC1
|
||||
members := `[{"id":"novox/a#1"}]`
|
||||
argv, _ := membersFlags(strings.Repeat(" ", membersInArgument) + members)
|
||||
defer removeMembersFile(argv)
|
||||
if len(argv) != 2 || argv[0] != "--members-file" {
|
||||
t.Fatalf("a long list gives %v", argv)
|
||||
}
|
||||
if st, err := os.Stat(argv[1]); err != nil || st.Mode().Perm() != 0o600 {
|
||||
t.Errorf("the file is %v, %v", st, err)
|
||||
}
|
||||
}
|
||||
@@ -154,6 +154,24 @@ type mergeWidthOf struct {
|
||||
// wideRebuild is how many modules a rebuild may take before the gate says so as a warning.
|
||||
var wideRebuild = 12
|
||||
|
||||
// changedPaths are the paths --changed (comma-separated) and --changed-file (one per line) name together.
|
||||
func changedPaths(list, file string) ([]string, error) {
|
||||
out := splitList(list)
|
||||
if file == "" {
|
||||
return out, nil
|
||||
}
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("--changed-file: %w", err)
|
||||
}
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
if line = strings.TrimSpace(line); line != "" {
|
||||
out = append(out, line)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// mergeGateCommand is `gate`:
|
||||
//
|
||||
// merge-gate --facts <file> --store <postgres> [--repository owner/repo --tree <dir>] [--changed a,b] [--json]
|
||||
@@ -167,6 +185,8 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
repository := set.String("repository", "", "owner/repository of the change")
|
||||
tree := set.String("tree", "", "the change's checkout: every module.json in it replaces the mesh's")
|
||||
changed := set.String("changed", "", "the paths the change touches, comma-separated, for its rebuild's width")
|
||||
changedFile := set.String("changed-file", "", "a file of the paths the change touches, one per line; the same as "+
|
||||
"--changed, for a list too long for an argument (novox/hq issue 1247)")
|
||||
group := set.String("group", "", "a delivery group's other heads, as JSON [{repository, tree, changed}], composed "+
|
||||
"with this one as one future state (novox/hq ADR 0239)")
|
||||
asJSON := set.Bool("json", false, "print the verdict as JSON")
|
||||
@@ -198,7 +218,11 @@ func mergeGateCommand(ctx context.Context, args []string) error {
|
||||
// verdict at all. What is said goes beside the report.
|
||||
os.Stdout = os.Stderr
|
||||
}
|
||||
v, err := judgeChange(ctx, mergeCheckInput{facts: f, repository: *repository, tree: *tree, changed: splitList(*changed),
|
||||
changedList, err := changedPaths(*changed, *changedFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
v, err := judgeChange(ctx, mergeCheckInput{facts: f, repository: *repository, tree: *tree, changed: changedList,
|
||||
group: others, admin: *admin, say: out, beside: os.Getenv("MESH_CHECK_BESIDE")})
|
||||
os.Stdout = stdout
|
||||
if err != nil {
|
||||
@@ -1120,6 +1144,9 @@ func throwawayStores(ctx context.Context, admin string) (*stores, func(), error)
|
||||
drop()
|
||||
return nil, nil, err
|
||||
}
|
||||
// The chain lives on the bus, and the gate has none: this mesh is raised for a moment, with a chain of its own in memory
|
||||
// that no other reads (issue 1121).
|
||||
open.inventory.UseThrowawayChain()
|
||||
for _, m := range provided {
|
||||
if err := open.inventory.Provide(ctx, m); err != nil {
|
||||
open.Close()
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTheGateJudgesAFileOfPathsAsItDoesTheSameListOfThem(t *testing.T) { // 1247/AC3
|
||||
paths := []string{"modules/a/x.go", "modules/a/y.go", "modules/b/z.go", "README.md"}
|
||||
file := filepath.Join(t.TempDir(), "changed.txt")
|
||||
if err := os.WriteFile(file, []byte(strings.Join(paths, "\n")+"\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fromFile, err := changedPaths("", file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fromList, err := changedPaths(strings.Join(paths, ","), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(fromFile, fromList) || len(fromFile) != len(paths) {
|
||||
t.Errorf("the file gives %v, the list %v", fromFile, fromList)
|
||||
}
|
||||
if _, err := changedPaths("", filepath.Join(t.TempDir(), "none")); err == nil {
|
||||
t.Error("a file that is not there is read as no paths")
|
||||
}
|
||||
}
|
||||
@@ -50,9 +50,17 @@ func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) {
|
||||
t.Cleanup(func() {
|
||||
stop()
|
||||
<-done
|
||||
lookingAtBatches.Wait()
|
||||
setWindowClock(nil)
|
||||
windowLook.Store(nil)
|
||||
})
|
||||
// The forge's merges are taken from a consumer made when the server starts serving: a merge published before it exists
|
||||
// is not delivered to it. With the chain read from the bus the set-up above is quick enough to lose that race, so the
|
||||
// publisher waits for the consumer.
|
||||
becomes(t, 10*time.Second, "the merges' consumer being made", func() bool {
|
||||
_, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName)
|
||||
return err == nil
|
||||
})
|
||||
return func(m link.SourceMoved) {
|
||||
t.Helper()
|
||||
body, err := json.Marshal(m)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// refusing is a bucket that refuses to write while it is told to.
|
||||
type refusing struct {
|
||||
chain.Backend
|
||||
refuse atomic.Bool
|
||||
}
|
||||
|
||||
func (r *refusing) Create(ctx context.Context, key string, value []byte) (uint64, error) {
|
||||
if r.refuse.Load() {
|
||||
return 0, nats.ErrTimeout
|
||||
}
|
||||
return r.Backend.Create(ctx, key, value)
|
||||
}
|
||||
|
||||
func (r *refusing) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if r.refuse.Load() {
|
||||
return 0, nats.ErrTimeout
|
||||
}
|
||||
return r.Backend.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
// **A merge whose plan cannot be saved is held, recorded nowhere, and recorded when it is handed over again** (ADR 0407
|
||||
// decision 3, issue 1092): hearMerge answers ErrTryAgain — the bus leaves the announcement unacknowledged and hands it
|
||||
// over again — and the merge is in no record until the plan holding it is written.
|
||||
func TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain(t *testing.T) {
|
||||
open := windowed(t)
|
||||
everyHeadWaits(t)
|
||||
asksWithPaths(t)
|
||||
ctx := t.Context()
|
||||
rollouts := &refusing{Backend: chain.NewMemory()}
|
||||
open.inventory.UseChain(chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil }))
|
||||
merge := catalogueMerge("cat1aaaaaaaa", "app", t0)
|
||||
|
||||
rollouts.refuse.Store(true)
|
||||
err := (following{open: open}).hearMerge(ctx, merge, t0.Add(time.Second))
|
||||
if !errors.Is(err, link.ErrTryAgain) {
|
||||
t.Fatalf("a merge whose plan could not be saved answered %v, which the bus takes as done", err)
|
||||
}
|
||||
if _, known, err := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); err != nil || known {
|
||||
t.Fatalf("the merge is kept though its plan was not: %v %v", known, err)
|
||||
}
|
||||
if _, bs := walks(t, open); len(bs) != 0 {
|
||||
t.Fatalf("a batch was kept though its write failed: %+v", bs)
|
||||
}
|
||||
|
||||
rollouts.refuse.Store(false)
|
||||
hear(t, open, merge, t0.Add(10*time.Second))
|
||||
_, bs := walks(t, open)
|
||||
if len(bs) != 1 {
|
||||
t.Fatalf("the merge handed over again made %d batch(es)", len(bs))
|
||||
}
|
||||
if m, known, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); !known || m.Plan != bs[0].ID {
|
||||
t.Fatalf("the merge is %+v %v, and the batch %s", m, known, bs[0].ID)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// **A merge whose plan could not be saved is held, never taken** (novox/hq ADR 0407 decision 3, issue 1092): the merge is
|
||||
// carried until its plan is saved, and the bus acknowledges the announcement only when the controller answers without
|
||||
// ErrTryAgain (link.Server.sourceMoved), so a failed save of any kind must answer it.
|
||||
func TestAFailureToSaveAMergesPlanIsHeldAndNotTaken(t *testing.T) {
|
||||
cause := errors.New("the plan for novox/mesh-catalog is not written: the plan was written by somebody else since it was read")
|
||||
held := unsaved(cause)
|
||||
if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) {
|
||||
t.Fatalf("a failed save answers %v", held)
|
||||
}
|
||||
if unsaved(nil) != nil {
|
||||
t.Fatal("a save that worked answers an error")
|
||||
}
|
||||
// Not the same as notNow, which holds a merge only for a store that is away.
|
||||
if errors.Is(notNow(cause), link.ErrTryAgain) {
|
||||
t.Fatal("notNow holds every failure; the premise of this test is gone")
|
||||
}
|
||||
}
|
||||
@@ -145,9 +145,15 @@ func (w *busWindow) Forget(ctx context.Context, batch string) {
|
||||
}
|
||||
}
|
||||
|
||||
// lookingAtBatches counts the looks under way, so a test that ends the window's clock can wait for the last of them before the
|
||||
// next test replaces what they read.
|
||||
var lookingAtBatches sync.WaitGroup
|
||||
|
||||
// lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches
|
||||
// whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once.
|
||||
func lookAtBatches(ctx context.Context, open *stores) {
|
||||
lookingAtBatches.Add(1)
|
||||
defer lookingAtBatches.Done()
|
||||
release, err := open.inventory.HoldPlans(ctx, true)
|
||||
if err != nil {
|
||||
fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err)
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1211/AC1 to AC3: the controller keeps a merge announced by a module that claims the delivery seat, whatever its
|
||||
// name, and takes no word from one that does not (novox/hq ADR 0437 §3). With no bus held, a word that passes the
|
||||
// claim is held for the bus (ErrTryAgain); a word that does not is refused before the bus is asked.
|
||||
func TestOnlyAModuleClaimingTheSeatIsTheDeliverysWordOnAMerge(t *testing.T) {
|
||||
entries := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("gitea"), namedOnly("mesh-delivery")}
|
||||
keeper := keepingDeliveryMerges{
|
||||
js: func() *broker.JetStream { return nil },
|
||||
catalog: func(context.Context) ([]inventory.Entry, error) { return entries, nil },
|
||||
}
|
||||
body := []byte(`{"owner":"novox","repo":"r","number":1,"head_sha":"abc","ask_id":"a","option":"approve"}`)
|
||||
for emitter, passes := range map[string]bool{
|
||||
"mesh-delivery-gate": true, // the claimant, under another name
|
||||
"gitea": false, // claims no seat
|
||||
"mesh-delivery": false, // named so, claiming no seat here
|
||||
} {
|
||||
err := keeper.KeepDeliveryMerge(context.Background(), emitter, body)
|
||||
if passes != errors.Is(err, link.ErrTryAgain) || (!passes && !errors.Is(err, link.ErrNotTheDeliverysWord)) {
|
||||
t.Errorf("%s: %v (passes the claim: %v)", emitter, err, passes)
|
||||
}
|
||||
}
|
||||
entries = []inventory.Entry{claimingTheSeat("mesh-delivery")}
|
||||
if err := keeper.KeepDeliveryMerge(context.Background(), "mesh-delivery", body); !errors.Is(err, link.ErrTryAgain) {
|
||||
t.Errorf("today's module, claiming the seat: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -926,13 +926,45 @@ func brokerReachableAt(ctx context.Context, inv *inventory.Inventory, known brok
|
||||
// A module that runs its machine's provisioner client and declares no broker is issued the client's account
|
||||
// alone (issue 917): the client speaks on the bus only as itself.
|
||||
func mayIssue(m catalogue.Manifest) error {
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads && !runsTheProvisionerClient(m) {
|
||||
processes := m.ProcessBuses()
|
||||
// A module whose own tools run in the machine's tool runner — mesh-mail's real shape — declares no
|
||||
// `broker` of its own at all (novox/hq ADR 0436 §2): its process's account is a different account,
|
||||
// delivered as its own secret below, so a process with a bus is reason enough to pass this check on
|
||||
// its own, before the module's own `broker` is asked for.
|
||||
if _, reads := m.OwnSecrets["broker"]; !reads && !runsTheProvisionerClient(m) && len(processes) == 0 {
|
||||
return fmt.Errorf("%s declares no own secret named broker, so there is nowhere to deliver "+
|
||||
"an account; a module that speaks on the bus declares \"own-secrets\": {\"broker\": <path>}", m.Module)
|
||||
}
|
||||
// And every process with a bus declaration (novox/hq ADR 0436 §2): its own account is delivered as the
|
||||
// module's own secret broker-<process id>, never the module's `broker`, so a process with a bus and no
|
||||
// such secret has nowhere to read an account either.
|
||||
for id := range processes {
|
||||
name := processSecretName(id)
|
||||
if _, reads := m.OwnSecrets[name]; !reads {
|
||||
return fmt.Errorf("%s's process %s declares bus, and declares no own secret named %s, so there "+
|
||||
"is nowhere to deliver its account; a process with a bus declares \"own-secrets\": "+
|
||||
"{%q: <path>} (novox/hq ADR 0436 §2)", m.Module, id, name, name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// processSecretName is the own secret a process's bus account is delivered in (novox/hq ADR 0436 §2):
|
||||
// broker-<process id>, never the module's own `broker`.
|
||||
func processSecretName(processID string) string { return "broker-" + processID }
|
||||
|
||||
// processPrincipalFor is the one process of m's bus declarations named processID, as `rollout mint` finds it
|
||||
// again from a KindProcess principal's bare id (novox/hq ADR 0436 §1, §2): false when the manifest no longer
|
||||
// declares it.
|
||||
func processPrincipalFor(m catalogue.Manifest, processID string) (broker.ProcessPrincipal, bool) {
|
||||
for _, pr := range inventory.ProcessesOf(m) {
|
||||
if pr.Process == processID {
|
||||
return pr, true
|
||||
}
|
||||
}
|
||||
return broker.ProcessPrincipal{}, false
|
||||
}
|
||||
|
||||
// issueOnTheNewBus gives an assigned module its credential on the bus being built.
|
||||
//
|
||||
// **Three things differ from a management call, and each is the point of the move.** The credential
|
||||
@@ -968,12 +1000,82 @@ func issueOnTheNewBus(ctx context.Context, inv *inventory.Inventory, m catalogue
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Every process with a bus declaration gets an account of its own (novox/hq ADR 0436 §1, §2), minted and
|
||||
// delivered whether or not the module itself speaks on the bus — its own secret is broker-<process id>,
|
||||
// never the module's `broker`.
|
||||
for _, pr := range inventory.ProcessesOf(m) {
|
||||
if err := issueProcess(ctx, inv, m, pr, node, busAddress, known, reachable); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !runsTheProvisionerClient(m) {
|
||||
return nil
|
||||
}
|
||||
return issueProvisionerClient(ctx, inv, m, node, known, reachable)
|
||||
}
|
||||
|
||||
// issueProcess gives one process its account: minted under its own name <node>.<module>.<process id>,
|
||||
// delivered as the module's own secret broker-<process id> in the provisioner client's form (novox/hq ADR
|
||||
// 0436 §2), handed to the vault as any covered account is, and its work-queue consumer or consumers
|
||||
// created where the bus can be reached — never the module's claims, since a process holds no seat.
|
||||
func issueProcess(ctx context.Context, inv *inventory.Inventory, m catalogue.Manifest, pr broker.ProcessPrincipal,
|
||||
node, busAddress string, known broker.Broker, reachable string) error {
|
||||
user := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}.Username()
|
||||
password, err := inv.MintBusPassword(ctx, inventory.BusUser{
|
||||
Username: user, Kind: inventory.BusProcess, Node: node, Module: m.Module,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := processSecretName(pr.Process)
|
||||
held, err := heldProcessAccount(known, reachable, node, m.Module, user, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := inv.AcceptSecretForModule(ctx, node, m.Module, name, string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := handIssuedToVault(ctx, inv, m, node, name, string(held)); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(pr.Queues) > 0 {
|
||||
if busAddress == "" {
|
||||
fmt.Printf(" %s's process %s consumes its work queue; its consumer is created when the bus is "+
|
||||
"reachable (`push --behind`, then `rollout mint` again is harmless)\n", m.Module, pr.Process)
|
||||
} else {
|
||||
js, err := broker.Dial(busAddress)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the credential is minted and the mesh cannot reach the bus to create "+
|
||||
"%s's process %s's work-queue consumer: %w", m.Module, pr.Process, err)
|
||||
}
|
||||
defer js.Close()
|
||||
who := broker.Principal{Kind: broker.KindProcess, Node: node, Module: m.Module, Process: pr.Process}
|
||||
for _, q := range pr.Queues {
|
||||
if err := js.EnsureConsumer(broker.QueueConsumer(who, q)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("bus user %s minted for %s's process %s, scoped to its bus declaration\n", user, m.Module, pr.Process)
|
||||
fmt.Printf(" sealed to %s as %s. It arrives with the next push — `push %s` to send it\n", node, name, node)
|
||||
return nil
|
||||
}
|
||||
|
||||
// heldProcessAccount is a process's account, in the provisioner client's form (novox/hq ADR 0436 §2):
|
||||
// {url, fingerprint, node, module, user, password} — never claims, since a process holds no seat (§3).
|
||||
func heldProcessAccount(known broker.Broker, reachable, node, module, user, password string) ([]byte, error) {
|
||||
return json.Marshal(struct {
|
||||
URL string `json:"url"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
}{URL: "nats://" + reachable, Fingerprint: known.Fingerprint, Node: node, Module: module, User: user,
|
||||
Password: password})
|
||||
}
|
||||
|
||||
// runsTheProvisionerClient says a module runs its machine's provisioner client, which reaches the bus on an
|
||||
// account of its own (novox/hq ADR 0349 clause 11, design 57 P3): it claims the seat and declares the own secret
|
||||
// that account is delivered in. The same test the bus's user list makes (inventory's declaredFor).
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// 1145/AC1 on a store: declarationWith itself sets the module of each resource and the build of each module for a
|
||||
// node recorded as reading them, and for no other. Skips without a store.
|
||||
func TestAComposedDeclarationCarriesModulesAndBuildsOnlyForANodeThatReadsThem(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
start := time.Now().Add(-time.Hour)
|
||||
manifest := resticFixture(false)
|
||||
manifest["module"], manifest["version"] = "restic", "1"
|
||||
raw, _ := json.Marshal(manifest)
|
||||
b := link.BuildResult{ID: link.NewBuildID(start), Repository: "novox/mesh-catalog", Path: "modules/restic",
|
||||
On: "anchor", Module: "restic", Commit: "c1", Manifest: raw,
|
||||
Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}
|
||||
if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, "restic"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
anchor, err := inv.NodeByName(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordReadsModules(ctx, anchor.ID, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantBuild, err := inv.BuildOf(ctx, "restic", "c1")
|
||||
if err != nil || wantBuild == "" {
|
||||
t.Fatalf("the build of restic at c1 is %q (%v)", wantBuild, err)
|
||||
}
|
||||
|
||||
declare := func(node string) sendable {
|
||||
t.Helper()
|
||||
plan, settings, err := planFor(ctx, open, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gens, err := generators(ctx, open)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := declarationWith(ctx, open, node, plan, settings, gens, Allocating)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
reads := bodyKeys(t, declare("anchor"))
|
||||
builds, _ := reads["builds"].(map[string]any)
|
||||
if builds["restic"] != wantBuild {
|
||||
t.Errorf("1145/AC1: anchor reads modules and was sent builds %v, want restic at %s", reads["builds"], wantBuild)
|
||||
}
|
||||
owned := 0
|
||||
for _, r := range reads["resources"].([]any) {
|
||||
r := r.(map[string]any)
|
||||
if strings.HasPrefix(r["id"].(string), "restic.") {
|
||||
owned++
|
||||
if r["module"] != "restic" {
|
||||
t.Errorf("1145/AC1: resource %v carries module %v, want restic", r["id"], r["module"])
|
||||
}
|
||||
}
|
||||
}
|
||||
if owned == 0 {
|
||||
t.Fatal("1145/AC1: no resource of restic was composed")
|
||||
}
|
||||
|
||||
plain := bodyKeys(t, declare("laptop"))
|
||||
if _, there := plain["builds"]; there {
|
||||
t.Errorf("1145/AC1: laptop never said it reads modules and was sent builds %v", plain["builds"])
|
||||
}
|
||||
for _, r := range plain["resources"].([]any) {
|
||||
if _, there := r.(map[string]any)["module"]; there {
|
||||
t.Fatalf("1145/AC1: laptop was sent a module on %v", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-host/validate"
|
||||
)
|
||||
|
||||
// The module of each resource and the build of each module (novox/hq issue 1145, design 47 release 1-2). The
|
||||
// composer's owner of an id and the module's build go on the wire only to a machine whose node-engine said it reads
|
||||
// them: an older one decodes strictly and refuses a key it does not know, whole.
|
||||
|
||||
func modulesDeclaration(reads bool) sendable {
|
||||
return sendable{
|
||||
Resources: []map[string]any{
|
||||
{"id": "web.app.conf", "type": "file", "path": "/etc/app.conf", "content": "x\n"},
|
||||
{"id": "web.app.data", "type": "file", "path": "/etc/data", "content": "d\n"},
|
||||
{"id": "mesh.guard", "type": "file", "path": "/etc/guard", "content": "g\n"}},
|
||||
moduleOf: map[string]string{"web.app.conf": "web.app", "web.app.data": "web.app"},
|
||||
buildIDs: map[string]string{"web.app": "build-7"},
|
||||
ReadsModules: reads,
|
||||
}
|
||||
}
|
||||
|
||||
// 1145/AC1
|
||||
func TestAMachineThatReadsModulesIsSentEachResourcesModuleAndEachModulesBuild(t *testing.T) {
|
||||
keys := bodyKeys(t, modulesDeclaration(true))
|
||||
builds, _ := keys["builds"].(map[string]any)
|
||||
if len(builds) != 1 || builds["web.app"] != "build-7" {
|
||||
t.Fatalf("1145/AC1: the body carries builds %v", keys["builds"])
|
||||
}
|
||||
want := map[string]any{"web.app.conf": "web.app", "web.app.data": "web.app", "mesh.guard": nil}
|
||||
for _, r := range keys["resources"].([]any) {
|
||||
r := r.(map[string]any)
|
||||
if r["module"] != want[r["id"].(string)] {
|
||||
t.Errorf("1145/AC1: resource %v carries module %v, want %v", r["id"], r["module"], want[r["id"].(string)])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1145/AC2: the body is byte for byte what it is today.
|
||||
func TestAMachineThatDoesNotReadModulesIsSentTheBodyItWasSentBefore(t *testing.T) {
|
||||
d := modulesDeclaration(false)
|
||||
got, err := d.Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := sendable{Resources: d.Resources}
|
||||
want, _ := before.Body()
|
||||
if string(got) != string(want) {
|
||||
t.Fatalf("1145/AC2: the body changed:\n%s\nwas\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1145/AC4: the node-engine's own validator, as the gate runs it, takes the two words.
|
||||
func TestTheNodeEnginesValidatorTakesAModulePerResourceAndABuildPerModule(t *testing.T) {
|
||||
raw, err := modulesDeclaration(true).Body()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var generic map[string]any
|
||||
if err := json.Unmarshal(raw, &generic); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if problems := validate.Declaration(raw); len(problems) != 0 {
|
||||
t.Fatalf("1145/AC4: refused: %q\n%s", problems, fmt.Sprint(generic))
|
||||
}
|
||||
}
|
||||
|
||||
// The build a module carries is the one its assignment names when that is the commit carried, else the one built from
|
||||
// the commit carried; none when neither is known.
|
||||
func TestTheBuildOfACarriedCommitIsTheAssignmentsOrTheOneBuiltFromIt(t *testing.T) {
|
||||
runs := map[string]chain.Ref{"a": {Build: "b-a", Commit: "c1"}, "b": {Build: "b-old", Commit: "c0"}}
|
||||
looked := map[string]string{"b/c2": "b-new"}
|
||||
got, err := wireBuilds(map[string]string{"a": "c1", "b": "c2", "c": "c3", "d": ""}, runs,
|
||||
func(module, commit string) (string, error) { return looked[module+"/"+commit], nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"a": "b-a", "b": "b-new"}
|
||||
if len(got) != len(want) || got["a"] != want["a"] || got["b"] != want["b"] {
|
||||
t.Fatalf("1145: builds %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -12,8 +13,10 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -44,6 +47,28 @@ type deliveryMerge struct {
|
||||
AskID string `json:"ask_id"`
|
||||
Option string `json:"option"`
|
||||
Heads map[string]string `json:"heads"`
|
||||
// Asker is the module that asked the ask, whose warrant is read from the router under that name (novox/hq ADR
|
||||
// 0437 §6): empty is today's module, mesh-delivery.
|
||||
Asker string `json:"asker,omitempty"`
|
||||
}
|
||||
|
||||
// askerOf is the asker whose warrant the merge is read under: the one the announcement names, mesh-delivery when it
|
||||
// names none.
|
||||
func (d deliveryMerge) askerOf() string {
|
||||
if d.Asker == "" {
|
||||
return theDelivery
|
||||
}
|
||||
return d.Asker
|
||||
}
|
||||
|
||||
// askerAccepted is whether the controller takes a warrant from this asker for a merge: the teller, and mesh-delivery
|
||||
// until the gate takes the seat over (novox/hq ADR 0437 §6). Any other registers no move.
|
||||
func askerAccepted(asker string) error {
|
||||
if asker == theDelivery || asker == sdkdelivery.TellerModule {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("the merge names the asker %q, and the controller takes a warrant only from %s and %s",
|
||||
asker, sdkdelivery.TellerModule, theDelivery)
|
||||
}
|
||||
|
||||
// parseDeliveryMerge reads an announcement, refusing one that names no pull request, head or ask.
|
||||
@@ -79,6 +104,10 @@ func mergeAct(ask, option string, heads map[string]string) asks.Act {
|
||||
// proved by the channel it came through, of this very ask as it was published (digest), bound to the heads the
|
||||
// announcement names, and those heads hold the head merged. Answers why it holds, or why not.
|
||||
func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, error) {
|
||||
asker := d.askerOf()
|
||||
if err := askerAccepted(asker); err != nil {
|
||||
return "", err
|
||||
}
|
||||
switch {
|
||||
case ask == nil || w == nil:
|
||||
return "", errors.New("the router holds no closed record of the ask the merge was made on")
|
||||
@@ -87,7 +116,7 @@ func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, er
|
||||
case d.Option != optionApprove:
|
||||
return "", fmt.Errorf("the merge was made on the option %q, not an Approve", d.Option)
|
||||
}
|
||||
option, err := w.For(theDelivery, *ask)
|
||||
option, err := w.For(asker, *ask)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -115,9 +144,26 @@ func movedOnApprove(d deliveryMerge, ask *asks.Ask, w *asks.Warrant) (string, er
|
||||
// keepingDeliveryMerges keeps each merge mesh-delivery announces in the controller's bucket for them.
|
||||
type keepingDeliveryMerges struct {
|
||||
js func() *broker.JetStream
|
||||
// catalog is the catalogue, whose modules claiming the delivery seat are the only ones whose word on a merge is
|
||||
// kept (novox/hq ADR 0437 §3).
|
||||
catalog func(ctx context.Context) ([]inventory.Entry, error)
|
||||
// kept, when set, is told each merge once it is kept: a first merge waiting for the word asks its builds then
|
||||
// (novox/hq ADR 0432, issue 1146).
|
||||
kept func(ctx context.Context, d deliveryMerge) error
|
||||
}
|
||||
|
||||
func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, body []byte) error {
|
||||
func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, emitter string, body []byte) error {
|
||||
if k.catalog == nil {
|
||||
return fmt.Errorf("%w: the catalogue is not read", link.ErrTryAgain)
|
||||
}
|
||||
entries, err := k.catalog(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
if !slices.Contains(claimantsOf(entries, catalogue.DeliverySeat), emitter) {
|
||||
return fmt.Errorf("%w: %q is not among %v", link.ErrNotTheDeliverysWord, emitter,
|
||||
claimantsOf(entries, catalogue.DeliverySeat))
|
||||
}
|
||||
d, err := parseDeliveryMerge(body)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -137,6 +183,9 @@ func (k keepingDeliveryMerges) KeepDeliveryMerge(ctx context.Context, body []byt
|
||||
if _, err := kv.Put(ctx, deliveryMergeKey(d.Owner, d.Repo, d.Number, d.HeadSHA), body); err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
if k.kept != nil {
|
||||
return k.kept(ctx, d)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -166,9 +215,9 @@ var keptDeliveryMerge = func(ctx context.Context, owner, repo string, number int
|
||||
return d, err == nil, err
|
||||
}
|
||||
|
||||
// routerAskOf is the router's record of one of mesh-delivery's asks: the ask as published and its warrant. A
|
||||
// routerAskOf is the router's record of one of the delivery's asks, under its asker: the ask as published and its warrant. A
|
||||
// variable so a test can hand it one.
|
||||
var routerAskOf = func(ctx context.Context, inv *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
var routerAskOf = func(ctx context.Context, inv catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return nil, nil, errors.New("only the serving controller reads the router's record")
|
||||
@@ -180,7 +229,7 @@ var routerAskOf = func(ctx context.Context, inv *inventory.Inventory, id string)
|
||||
if bucket == "" {
|
||||
return nil, nil, errors.New("no module declares the operator channel's records")
|
||||
}
|
||||
_, ask, w, err := readRouterEntry(ctx, serving.Conn(), bucket, theDelivery, id)
|
||||
_, ask, w, err := readRouterEntry(ctx, serving.Conn(), bucket, asker, id)
|
||||
return ask, w, err
|
||||
}
|
||||
|
||||
@@ -216,7 +265,7 @@ var errNotAMerge = errors.New("the build is not of a merge")
|
||||
// (ADR 0420 §7): the build is of a merge, that merge is a pull request mesh-delivery announced it made, and the
|
||||
// ask it made it on holds an Approve by the operator of that head. Answers why it was accepted; an error says why
|
||||
// it was not, and the refusal of ADR 0266 stands.
|
||||
func approvedMove(ctx context.Context, inv *inventory.Inventory, forms *sourceForms, buildID string,
|
||||
func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, buildID string,
|
||||
built inventory.Source) (string, error) {
|
||||
r, found, err := inv.BuildRequestByID(ctx, buildID)
|
||||
if err != nil {
|
||||
@@ -236,24 +285,42 @@ func approvedMove(ctx context.Context, inv *inventory.Inventory, forms *sourceFo
|
||||
if !known {
|
||||
return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, r.Commit)
|
||||
}
|
||||
k := announcedOf(merged)
|
||||
return approvedAnnounced(ctx, inv, owner, name, announcedOf(merged))
|
||||
}
|
||||
|
||||
// errWordNotAnApprove says the merge cannot be shown to be the operator's Approve of its head: its announcement names no
|
||||
// pull request, or the word on record is not that Approve.
|
||||
var errWordNotAnApprove = errors.New("the word on record is no Approve of the head merged")
|
||||
|
||||
// errNotAnnounced says mesh-delivery's word on a merge is not kept (yet): the merge may have been made by someone else,
|
||||
// or its word may still be on its way.
|
||||
var errNotAnnounced = errors.New("mesh-delivery announced no such merge")
|
||||
|
||||
// approvedAnnounced is the warrant of the merge the forge announced as k: mesh-delivery announced it, and the ask it made
|
||||
// it on holds the operator's Approve of that head. Why it holds, or why not. Shared by the approved move of a known
|
||||
// module and the first build of a new repository's modules (ADR 0420 §7, ADR 0432).
|
||||
func approvedAnnounced(ctx context.Context, inv catalogueReader, owner, name string, k keptMerge) (string, error) {
|
||||
if k.Number <= 0 || k.HeadSHA == "" {
|
||||
return "", errors.New("the forge's announcement of the merge names no pull request or head")
|
||||
return "", fmt.Errorf("%w: the forge's announcement of the merge names no pull request or head", errWordNotAnApprove)
|
||||
}
|
||||
d, kept, err := keptDeliveryMerge(ctx, owner, name, k.Number, k.HeadSHA)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !kept {
|
||||
return "", fmt.Errorf("mesh-delivery announced no merge of %s/%s#%d at %.8s", owner, name, k.Number, k.HeadSHA)
|
||||
return "", fmt.Errorf("%w: %s/%s#%d at %.8s", errNotAnnounced, owner, name, k.Number, k.HeadSHA)
|
||||
}
|
||||
ask, w, err := routerAskOf(ctx, inv, d.AskID)
|
||||
// The warrant is read under the asker the announcement names, and under none the controller does not take.
|
||||
if err := askerAccepted(d.askerOf()); err != nil {
|
||||
return "", fmt.Errorf("%w: %w", errWordNotAnApprove, err)
|
||||
}
|
||||
ask, w, err := routerAskOf(ctx, inv, d.askerOf(), d.AskID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
why, err := movedOnApprove(d, ask, w)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return "", fmt.Errorf("%w: %w", errWordNotAnApprove, err)
|
||||
}
|
||||
return fmt.Sprintf("%s/%s#%d merged at %.8s on %s", owner, name, k.Number, k.HeadSHA, why), nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The controller reads a merge's warrant under the asker the announcement names, and accepts only the teller and
|
||||
// today's mesh-delivery (novox/hq ADR 0437 §6, issue 1213). The teller's name is the SDK's.
|
||||
|
||||
// askedBy is an approved merge whose announcement names an asker ("" names none) and whose warrant is that asker's.
|
||||
func askedBy(t *testing.T, named, warrantOf string) anApprovedMerge {
|
||||
t.Helper()
|
||||
a := approvedMerge(t)
|
||||
a.merge.Asker = named
|
||||
a.warrant.Asker = warrantOf
|
||||
return a
|
||||
}
|
||||
|
||||
// 1213/AC1: an announcement naming the teller, and the teller's warrant of the operator's verified Approve of the
|
||||
// merge's heads: the move is registered (the warrant is read from the router under the teller).
|
||||
func TestAMergeNamingTheTellerIsMovedOnTheTellersWarrant(t *testing.T) {
|
||||
a := askedBy(t, sdkdelivery.TellerModule, sdkdelivery.TellerModule)
|
||||
if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err != nil {
|
||||
t.Fatalf("the teller's warrant of the operator's Approve was refused: %v", err)
|
||||
}
|
||||
was := routerAskOf
|
||||
t.Cleanup(func() { routerAskOf = was })
|
||||
wasKept := keptDeliveryMerge
|
||||
t.Cleanup(func() { keptDeliveryMerge = wasKept })
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
var readUnder string
|
||||
routerAskOf = func(_ context.Context, _ catalogueReader, asker, _ string) (*asks.Ask, *asks.Warrant, error) {
|
||||
readUnder = asker
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err != nil {
|
||||
t.Fatalf("the merge was not moved on the teller's warrant: %v", err)
|
||||
}
|
||||
if readUnder != sdkdelivery.TellerModule {
|
||||
t.Errorf("the warrant was read under %q", readUnder)
|
||||
}
|
||||
}
|
||||
|
||||
// 1213/AC2: an announcement naming mesh-delivery, or none (today's module): read under mesh-delivery, registered.
|
||||
func TestAMergeNamingNoAskerIsMovedOnMeshDeliverysWarrant(t *testing.T) {
|
||||
for _, named := range []string{"", theDelivery} {
|
||||
a := askedBy(t, named, theDelivery)
|
||||
if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err != nil {
|
||||
t.Errorf("asker %q: refused: %v", named, err)
|
||||
}
|
||||
}
|
||||
was := routerAskOf
|
||||
wasKept := keptDeliveryMerge
|
||||
t.Cleanup(func() { routerAskOf, keptDeliveryMerge = was, wasKept })
|
||||
a := askedBy(t, "", theDelivery)
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
var readUnder string
|
||||
routerAskOf = func(_ context.Context, _ catalogueReader, asker, _ string) (*asks.Ask, *asks.Warrant, error) {
|
||||
readUnder = asker
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if readUnder != theDelivery {
|
||||
t.Errorf("with no asker named the warrant was read under %q, not %q", readUnder, theDelivery)
|
||||
}
|
||||
}
|
||||
|
||||
// 1213/AC3: any other asker registers no move, and the reason names the asker refused; the router is not asked.
|
||||
func TestAMergeNamingAnyOtherAskerMovesNothingAndNamesIt(t *testing.T) {
|
||||
was := routerAskOf
|
||||
wasKept := keptDeliveryMerge
|
||||
t.Cleanup(func() { routerAskOf, keptDeliveryMerge = was, wasKept })
|
||||
for _, other := range []string{sdkdelivery.GateModule, "mesh-controller", "unknown-module"} {
|
||||
a := askedBy(t, other, other) // even a warrant of that very asker
|
||||
_, err := movedOnApprove(a.merge, &a.ask, &a.warrant)
|
||||
if err == nil || !strings.Contains(err.Error(), other) {
|
||||
t.Errorf("asker %q: %v", other, err)
|
||||
}
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, true, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
t.Errorf("the router was read under the refused asker %q", other)
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err == nil ||
|
||||
!strings.Contains(err.Error(), other) {
|
||||
t.Errorf("asker %q, approved as announced: %v", other, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1213/AC4: a warrant whose own asker differs from the asker read, or whose heads differ from the head merged, moves nothing.
|
||||
func TestAWarrantOfAnotherAskerOrOtherHeadsMovesNothing(t *testing.T) {
|
||||
a := askedBy(t, sdkdelivery.TellerModule, theDelivery) // read under the teller, but the warrant says mesh-delivery
|
||||
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||
t.Errorf("a warrant of another asker was accepted: %s", why)
|
||||
}
|
||||
a = askedBy(t, "", sdkdelivery.TellerModule) // today's module named, the teller's warrant
|
||||
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||
t.Errorf("the teller's warrant under mesh-delivery was accepted: %s", why)
|
||||
}
|
||||
a = askedBy(t, sdkdelivery.TellerModule, sdkdelivery.TellerModule)
|
||||
a.merge.HeadSHA = "d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4"
|
||||
if why, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil {
|
||||
t.Errorf("a head the Approve did not cover was accepted: %s", why)
|
||||
}
|
||||
}
|
||||
|
||||
// 1213/AC5: the announcement's asker is read and kept with the rest of it, under the JSON name asker.
|
||||
func TestTheAnnouncementsAskerIsReadAndKept(t *testing.T) {
|
||||
body := `{"owner":"novox","repo":"r","number":1,"head_sha":"abc","ask_id":"a","option":"approve","asker":"mesh-delivery-teller"}`
|
||||
d, err := parseDeliveryMerge([]byte(body))
|
||||
if err != nil || d.Asker != sdkdelivery.TellerModule {
|
||||
t.Fatalf("read %+v, %v", d, err)
|
||||
}
|
||||
none, err := parseDeliveryMerge([]byte(strings.Replace(body, `,"asker":"mesh-delivery-teller"`, "", 1)))
|
||||
if err != nil || none.Asker != "" {
|
||||
t.Fatalf("an announcement naming none read as %+v, %v", none, err)
|
||||
}
|
||||
}
|
||||
|
||||
// announcedPull is the forge's announcement of a merge of a pull request at a head.
|
||||
func announcedPull(number int, head string) keptMerge {
|
||||
var k keptMerge
|
||||
k.Number, k.HeadSHA = number, head
|
||||
return k
|
||||
}
|
||||
@@ -74,7 +74,7 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The events consumer exists, from now on, before the word is published: Receive would make it too late.
|
||||
@@ -90,7 +90,7 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
|
||||
|
||||
// mesh-delivery says it merged the pull request, and the controller keeps what it said.
|
||||
body, _ := json.Marshal(a.merge)
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubject, body); err != nil {
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the merge being kept", func() bool {
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **The operator's Approve of a merge registers a new module from its repository** (novox/hq ADR 0432, issue 1146).
|
||||
// The mesh as the decision reads it, with nothing registered from the repository, and the build request and merge the
|
||||
// controller kept.
|
||||
type anEmptyMesh struct {
|
||||
request inventory.BuildRequest
|
||||
asked bool
|
||||
merge inventory.BatchedMerge
|
||||
}
|
||||
|
||||
func (m *anEmptyMesh) Catalogue(context.Context) (map[string]catalogue.Manifest, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (m *anEmptyMesh) SourceOf(context.Context, string) (inventory.Source, error) {
|
||||
return inventory.Source{}, inventory.ErrNoSuchModule
|
||||
}
|
||||
func (m *anEmptyMesh) SourceIdentity(context.Context, string) (int64, error) { return 0, nil }
|
||||
func (m *anEmptyMesh) Catalogued(context.Context) ([]inventory.Entry, error) { return nil, nil }
|
||||
func (m *anEmptyMesh) BuildRequestByID(_ context.Context, id string) (inventory.BuildRequest, bool, error) {
|
||||
return m.request, m.asked && id == m.request.ID, nil
|
||||
}
|
||||
func (m *anEmptyMesh) MergeOf(context.Context, string, string) (inventory.BatchedMerge, bool, error) {
|
||||
return m.merge, true, nil
|
||||
}
|
||||
|
||||
// aDeliveryMergeOf is the Approved merge of a pull request of novox/mesh-delivery, its heads keyed by that repository,
|
||||
// and the ask and warrant bound to exactly those heads (the digest the Approve left covers them).
|
||||
func aDeliveryMergeOf(t *testing.T, head string, covered map[string]string) anApprovedMerge {
|
||||
t.Helper()
|
||||
a := approvedMerge(t)
|
||||
a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-delivery", 7, head
|
||||
a.merge.Heads = covered
|
||||
for n := range a.ask.Options {
|
||||
d, err := asks.ActDigest(mergeAct(a.merge.AskID, a.ask.Options[n].ID, covered))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.ask.Options[n].Binds = d
|
||||
}
|
||||
a.warrant.AskDigest = a.ask.Digest()
|
||||
return a
|
||||
}
|
||||
|
||||
func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwise(t *testing.T) {
|
||||
const commit, head = "c0ffee0123456789", "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"
|
||||
built := inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
change func(*anApprovedMerge)
|
||||
request string // "merge", or "build" through a verb
|
||||
unkept bool
|
||||
forge any // what the forge says of the repository; nil is protected
|
||||
built *inventory.Source
|
||||
accepted bool
|
||||
ac string
|
||||
}{
|
||||
{what: "the Approve of the merged head, a protected trunk on the forge", request: "merge", accepted: true, ac: "1146/AC1"},
|
||||
{what: "a build asked through a verb", request: "build", ac: "1146/AC2"},
|
||||
{what: "a merge with no warrant", request: "merge", unkept: true, ac: "1146/AC2"},
|
||||
{what: "a merged head the Approve did not cover", request: "merge", ac: "1146/AC2", change: func(a *anApprovedMerge) {
|
||||
*a = aDeliveryMergeOf(t, head, map[string]string{
|
||||
"novox/mesh-delivery@eeeeeeeeeeee": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"})
|
||||
}},
|
||||
{what: "heads other than the ones the Approve was bound to", request: "merge", ac: "1146/AC2", change: func(a *anApprovedMerge) {
|
||||
a.merge.Heads = map[string]string{"novox/mesh-delivery@eeeeeeeeeeee": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"}
|
||||
}},
|
||||
{what: "a repository not on the mesh's forge", request: "merge", ac: "1146/AC3",
|
||||
built: &inventory.Source{Repository: "http://elsewhere.example/x/y.git"}},
|
||||
{what: "a forge that cannot be asked", request: "merge", ac: "1146/AC3", forge: errors.New("connection refused")},
|
||||
{what: "an unprotected trunk", request: "merge", ac: "1146/AC3", forge: forgeFacts{ID: 9, Guarded: false, Why: "direct push"}},
|
||||
} {
|
||||
t.Run(c.ac+" "+c.what, func(t *testing.T) {
|
||||
a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head})
|
||||
if c.change != nil {
|
||||
c.change(&a)
|
||||
}
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
return a.merge, !c.unkept, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
return &a.ask, &a.warrant, nil
|
||||
}
|
||||
if c.forge != nil {
|
||||
theForge.Store("novox/mesh-delivery", c.forge)
|
||||
t.Cleanup(func() { theForge.Delete("novox/mesh-delivery") })
|
||||
}
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||
"head_sha": head, "merge_commit_sha": commit})
|
||||
mesh := &anEmptyMesh{asked: true, merge: inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main",
|
||||
Commit: commit, Merged: time.Now(), Event: event},
|
||||
request: inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "",
|
||||
Ref: "main", Commit: commit, For: c.request}}
|
||||
forms := &sourceForms{bases: map[string]string{}, base: func(seat string) string {
|
||||
if seat == "git" {
|
||||
return "http://forge.internal:20000"
|
||||
}
|
||||
return ""
|
||||
}}
|
||||
src := built
|
||||
if c.built != nil {
|
||||
src = *c.built
|
||||
}
|
||||
id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-view", src, "build-1", "", "main")
|
||||
switch {
|
||||
case c.accepted && err != nil:
|
||||
t.Fatalf("%s: the Approved merge's new module was refused: %v", c.ac, err)
|
||||
case c.accepted && id == 0:
|
||||
t.Fatalf("%s: registered with no forge identity to record", c.ac)
|
||||
case !c.accepted && !errors.Is(err, errNotItsSource):
|
||||
t.Fatalf("%s: %s was not refused: %v", c.ac, c.what, err)
|
||||
case !c.accepted && c.ac == "1146/AC2" &&
|
||||
(!strings.Contains(err.Error(), "controller's terminal") || !strings.Contains(err.Error(), "Approve")):
|
||||
t.Fatalf("%s: the refusal does not name both ways in: %v", c.ac, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A second new module from a repository whose first was registered is the case ADR 0266 §44 already has —
|
||||
// "alongside": registered through a verb with no warrant asked of the router (1146/AC5).
|
||||
type aMeshWithOneAlready struct{ anEmptyMesh }
|
||||
|
||||
func (m *aMeshWithOneAlready) Catalogued(context.Context) ([]inventory.Entry, error) {
|
||||
e := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}}
|
||||
e.Manifest.Module = "mesh-delivery-view"
|
||||
return []inventory.Entry{e}, nil
|
||||
}
|
||||
|
||||
func TestASecondNewModuleFromARepositoryIsJudgedAlongsideAndAsksNoWarrant(t *testing.T) {
|
||||
wasKept, wasAsk := keptDeliveryMerge, routerAskOf
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) {
|
||||
t.Error("1146/AC5: the announced merges were asked for a module judged alongside")
|
||||
return deliveryMerge{}, false, nil
|
||||
}
|
||||
routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) {
|
||||
t.Error("1146/AC5: the router was asked for a module judged alongside")
|
||||
return nil, nil, nil
|
||||
}
|
||||
mesh := &aMeshWithOneAlready{anEmptyMesh{asked: true,
|
||||
request: inventory.BuildRequest{ID: "build-2", Repository: "novox/mesh-delivery", Seat: "git", For: "build"}}}
|
||||
forms := &sourceForms{bases: map[string]string{}, base: func(string) string { return "http://forge.internal:20000" }}
|
||||
built := inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}
|
||||
if id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-watchman", built, "build-2", "", "main"); err != nil || id == 0 {
|
||||
t.Fatalf("1146/AC5: a second module from a repository the catalogue builds from was refused (id %d): %v", id, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1146/AC4: a new module from a repository nothing registered is built from, whole against a real bus and the store
|
||||
// (novox/hq ADR 0432, issue 1146; the warrant is read as for a move, ADR 0420 §7, issue 1025): mesh-delivery's word that it
|
||||
// merged a pull request is published on the subject the controller consumes and kept in its bucket; the router's
|
||||
// record of the ask is read back from the router's bucket by a direct get; and the build of a module from the
|
||||
// repository that merge was made in is registered from there only when that record holds the operator's Approve of
|
||||
// the merged head. Store-backed: skips without one.
|
||||
func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
what string
|
||||
answer bool // the router holds the operator's Approve of the merged heads
|
||||
moved bool
|
||||
}{
|
||||
{what: "the router holds the operator's Approve", answer: true, moved: true},
|
||||
{what: "the router holds no record of the ask"},
|
||||
} {
|
||||
t.Run(c.what, func(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
// The module declaring the router's records, as the controller finds them.
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1",
|
||||
DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}},
|
||||
inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
a := aDeliveryMergeOf(t, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1",
|
||||
map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"})
|
||||
if c.answer {
|
||||
record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant})
|
||||
if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, noListener{})
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The events consumer exists, from now on, before the word is published: Receive would make it too late.
|
||||
if err := broker.AssertServingConsumers(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serving, stop := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); _ = server.Serve(serving) }()
|
||||
t.Cleanup(func() { stop(); <-done })
|
||||
servingBus.Store(js)
|
||||
t.Cleanup(func() { servingBus.Store(nil) })
|
||||
|
||||
// mesh-delivery says it merged the pull request, and the controller keeps what it said.
|
||||
body, _ := json.Marshal(a.merge)
|
||||
if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
becomes(t, 15*time.Second, "the merge being kept", func() bool {
|
||||
_, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, a.merge.HeadSHA)
|
||||
return err == nil && kept
|
||||
})
|
||||
|
||||
// The forge's merge of that pull request, as the controller heard it, and the build it asked for.
|
||||
const commit = "c0ffee0123456789"
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-delivery", Seat: "git",
|
||||
Path: "modules/mesh-delivery-view", Ref: "main", Commit: commit, For: "merge"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := onTrunk(id, "novox/mesh-delivery", "git", "modules/mesh-delivery-view", map[string]any{"module": "mesh-delivery-view", "version": "2"})
|
||||
moved.Commit = commit
|
||||
_, _, err = takeIn(ctx, inv, moved)
|
||||
src, _ := inv.SourceOf(ctx, "mesh-delivery-view")
|
||||
if c.moved && (err != nil || src.Repository != "novox/mesh-delivery") {
|
||||
t.Fatalf("the Approved move was not registered from its repository: %v (built from %q)", err, src.Repository)
|
||||
}
|
||||
if !c.moved && (!errors.Is(err, errNotItsSource) || src.Repository != "") {
|
||||
t.Fatalf("a new module with no Approve was not refused as it is: %v (built from %q)", err, src.Repository)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -50,7 +50,7 @@ func nodeCommand(ctx context.Context, args []string) error {
|
||||
return addNode(ctx, inv, args[1:])
|
||||
|
||||
case "list":
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
nodes, err := open.heardNodes(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -755,22 +755,8 @@ func consumerReset(ctx context.Context, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// heardFrom says when a node was last heard from, in a form somebody can act on.
|
||||
//
|
||||
// "never" and "an hour ago" are different answers and are kept different. A node that has never
|
||||
// spoken did not finish joining; a node last heard from an hour ago is running an hour-old
|
||||
// picture of the mesh.
|
||||
func heardFrom(n inventory.Node) string {
|
||||
silent, ever := n.Silent()
|
||||
switch {
|
||||
case !ever:
|
||||
return "never spoken"
|
||||
case silent > SilentFor:
|
||||
return "out of touch " + roughly(silent)
|
||||
default:
|
||||
return "here"
|
||||
}
|
||||
}
|
||||
// heardFrom is heardFromAt for now.
|
||||
func heardFrom(n inventory.Node) string { return heardFromAt(n, time.Now()) }
|
||||
|
||||
// roughly is a duration a person reads rather than parses.
|
||||
func roughly(d time.Duration) string {
|
||||
@@ -795,7 +781,7 @@ func roughly(d time.Duration) string {
|
||||
// assignment carries `card1-DP-1`, and a person composing settings for that machine needs it.
|
||||
func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
inv := stored.inventory
|
||||
node, err := inv.NodeByName(ctx, name)
|
||||
node, err := stored.heardNode(ctx, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -929,6 +915,12 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
}
|
||||
if len(assigned) > 0 {
|
||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||
// And what each runs, as its assignment names it (novox/hq ADR 0408, issue 1095).
|
||||
if on, err := inv.AssignmentsOn(ctx, name); err == nil {
|
||||
for _, line := range runLines(on) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
|
||||
// reason its condition says. Not computable is said, never read as "all applied".
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// 1146: a merge's ask that failed is kept under an id the bus takes as a key, for a module at the repository's root
|
||||
// (".") as for one in a folder.
|
||||
func TestTheIdOfABuildNotAskedIsAKeyTheBusTakes(t *testing.T) {
|
||||
for dir, want := range map[string]string{
|
||||
".": "not-asked-0177ea7a",
|
||||
"modules/mesh-delivery-view": "not-asked-0177ea7a-modules-mesh-delivery-view",
|
||||
} {
|
||||
id := notAskedID("0177ea7a00000000", dir)
|
||||
if id != want {
|
||||
t.Errorf("notAskedID(%q) = %q, want %q", dir, id, want)
|
||||
}
|
||||
if !chain.ValidKey("req." + id) {
|
||||
t.Errorf("notAskedID(%q) = %q: the bus takes no key req.%s", dir, id, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The controller finds the delivery's owner by the claim of the seat mesh-delivery, never by the module's name
|
||||
// (novox/hq ADR 0437 §3, issue 1210). The gate is a module named mesh-delivery-gate; this is only a fixture name.
|
||||
|
||||
func claimingTheSeat(module string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: module,
|
||||
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
|
||||
}
|
||||
|
||||
func namedOnly(module string) inventory.Entry {
|
||||
return inventory.Entry{Manifest: catalogue.Manifest{Module: module}, On: []string{"anchor"}}
|
||||
}
|
||||
|
||||
// 1210/AC1: a module claiming the seat is on the controller's own path whatever its name, awaiting the delivery's
|
||||
// word as today's module does; a module named mesh-delivery that claims no seat is not on it.
|
||||
func TestTheModuleClaimingTheSeatIsOnTheControllersOwnPath(t *testing.T) {
|
||||
gate := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("mesh-delivery")}
|
||||
own := ownPathOf(gate)
|
||||
if _, on := own["mesh-delivery-gate"]; !on {
|
||||
t.Error("the module claiming the seat is not on the controller's own path")
|
||||
}
|
||||
if _, on := own["mesh-delivery"]; on {
|
||||
t.Error("a module named mesh-delivery that claims no seat is on the controller's own path")
|
||||
}
|
||||
if d := awaitsFor(gate, []string{"gitea", "plex"}); d == nil || d.Awaits != catalogue.DeliverySeat {
|
||||
t.Errorf("an ordinary walk awaits %+v with the gate holding the seat", d)
|
||||
}
|
||||
if d := awaitsFor(gate, []string{"mesh-delivery-gate", "gitea"}); d != nil {
|
||||
t.Errorf("a walk moving the gate awaits %+v; the gate cannot gate itself", d)
|
||||
}
|
||||
if d := awaitsFor(gate, []string{"mesh-delivery", "gitea"}); d == nil {
|
||||
t.Error("a walk moving a module named mesh-delivery that claims no seat starts at once")
|
||||
}
|
||||
if !ownWalk(walkOf("novox/one", "notes", "mesh-delivery-gate"), own) || ownWalk(walkOf("novox/one", "notes", "mesh-delivery"), own) {
|
||||
t.Error("a walk is on the own path by the claim of the module it plans")
|
||||
}
|
||||
if ownPath([]inventory.Entry{claimingTheSeat("mesh-delivery-gate")}) != true || ownPath([]inventory.Entry{namedOnly("mesh-delivery")}) {
|
||||
t.Error("a merge touches the own path by the claim of the module it touches")
|
||||
}
|
||||
if classOfModules([]string{"mesh-delivery-gate"}, gate) != inventory.ClassCore {
|
||||
t.Error("the module claiming the seat is not core")
|
||||
}
|
||||
}
|
||||
|
||||
// 1210/AC2: the merge gate's rule of the module claiming the seat is one reading; a module claiming no delivery
|
||||
// seat gets its ordinary rule.
|
||||
func TestTheModuleClaimingTheSeatGetsTheDeliverysGateRule(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
claimant := claimingTheSeat("mesh-delivery-gate").Manifest
|
||||
r := gateRuleOf("mesh-delivery-gate", claimant)
|
||||
if r.passes != 1 || r.within != 0 || !strings.HasPrefix(r.said, "mesh-delivery's rule: one reading (novox/hq ADR 0351)") {
|
||||
t.Errorf("the gate's rule is %+v", r)
|
||||
}
|
||||
r = gateRuleOf("mesh-delivery", catalogue.Manifest{Module: "mesh-delivery"})
|
||||
if !strings.HasPrefix(r.said, "a leaf module's rule") {
|
||||
t.Errorf("a module named mesh-delivery claiming no seat gets %q", r.said)
|
||||
}
|
||||
}
|
||||
|
||||
// 1210/AC3: with both today's module and the gate claiming the seat (R3d's group), both are on the own path and
|
||||
// both get the delivery's rule.
|
||||
func TestBothClaimantsAreOnTheOwnPathAndGetTheRule(t *testing.T) {
|
||||
withTheGatesTimes(t)
|
||||
both := []inventory.Entry{claimingTheSeat("mesh-delivery"), claimingTheSeat("mesh-delivery-gate")}
|
||||
own := ownPathOf(both)
|
||||
for _, m := range []string{"mesh-delivery", "mesh-delivery-gate"} {
|
||||
if _, on := own[m]; !on {
|
||||
t.Errorf("%s is not on the own path", m)
|
||||
}
|
||||
if r := gateRuleOf(m, claimingTheSeat(m).Manifest); r.passes != 1 || !strings.HasPrefix(r.said, "mesh-delivery's rule") {
|
||||
t.Errorf("%s gets %+v", m, r)
|
||||
}
|
||||
if d := awaitsFor(both, []string{m, "gitea"}); d != nil {
|
||||
t.Errorf("a walk moving %s awaits %+v", m, d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// theDeliveryClaims is a catalogue whose one module claiming the seat is today's mesh-delivery, for the tests that
|
||||
// announce a merge from it.
|
||||
func theDeliveryClaims(context.Context) ([]inventory.Entry, error) {
|
||||
return []inventory.Entry{claimingTheSeat("mesh-delivery")}, nil
|
||||
}
|
||||
@@ -491,8 +491,7 @@ func settlingPending(ctx context.Context, open *stores) {
|
||||
|
||||
// settlePending is one tick: claims left by a controller that stopped are settled from what the mesh holds;
|
||||
// a module registered meanwhile is assigned; a build heard and not registered, or silent past its bound,
|
||||
// ends its assignment; ended ones are raised as conditions and cleared once answered; ended rows older than
|
||||
// KeptFor are deleted. Returns what it did.
|
||||
// ends its assignment; ended ones are raised as conditions and cleared once answered. Returns what it did.
|
||||
func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
||||
inv := open.inventory
|
||||
var said []string
|
||||
@@ -565,11 +564,6 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string {
|
||||
}
|
||||
|
||||
said = append(said, raisePendingEnded(ctx, inv)...)
|
||||
if n, err := inv.ForgetEndedPending(ctx, now); err != nil {
|
||||
say("ended pending assignments could not be deleted: %v", err)
|
||||
} else if n > 0 {
|
||||
say("deleted %d pending assignment(s) ended more than %s ago", n, inventory.KeptFor)
|
||||
}
|
||||
return said
|
||||
}
|
||||
|
||||
@@ -585,7 +579,7 @@ func containsString(xs []string, x string) bool {
|
||||
// pendingObservation is the condition for a pending assignment that was not made: one per pending
|
||||
// assignment, its row's id the last part of its id, so one row's condition is never another's.
|
||||
func pendingObservation(p inventory.PendingAssignment) conditions.Observation {
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%d", p.Node, p.Module, p.ID),
|
||||
return conditions.Observation{Scope: conditions.ScopeMachine, ID: fmt.Sprintf("%s.%s.%s", p.Node, p.Module, p.ID),
|
||||
Token: kindPendingEnded, Kind: kindPendingEnded, Machine: p.Node, Severity: conditions.Warning,
|
||||
Resolver: conditions.ResolverOperator, Source: "pending assignments",
|
||||
Summary: p.Note}
|
||||
@@ -671,8 +665,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
|
||||
if err != nil {
|
||||
return []string{fmt.Sprintf("the open conditions could not be read to clear orphaned ones: %v", err)}
|
||||
}
|
||||
byID := map[int64]string{}
|
||||
var ids []int64
|
||||
byID := map[string]string{}
|
||||
var ids []string
|
||||
for _, c := range open {
|
||||
if c.Kind != kindPendingEnded {
|
||||
continue
|
||||
@@ -682,8 +676,8 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
var id int64
|
||||
if _, err := fmt.Sscan(parts[len(parts)-2], &id); err != nil {
|
||||
id := parts[len(parts)-2]
|
||||
if !strings.HasPrefix(id, "p-") {
|
||||
continue
|
||||
}
|
||||
byID[id] = c.Key
|
||||
@@ -701,7 +695,7 @@ func clearOrphaned(ctx context.Context, keeper *conditions.Keeper, inv *inventor
|
||||
if known[id] {
|
||||
continue
|
||||
}
|
||||
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed"); err != nil {
|
||||
if _, err := keeper.Clear(ctx, key, "its pending assignment is no longer on record: its machine was removed, or the module was assigned and the wait taken back"); err != nil {
|
||||
said = append(said, fmt.Sprintf("the condition %s could not be cleared: %v", key, err))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -542,23 +542,26 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
|
||||
t.Fatalf("a module registered meanwhile: %+v", p)
|
||||
}
|
||||
|
||||
// Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is
|
||||
// still open (second review, bug B).
|
||||
// Ended waits are not deleted, for the chain removes no key (ADR 0408 section 4): after KeptFor they are no longer listed, open
|
||||
// ones never are left out, nor one whose condition is still open (second review, bug B).
|
||||
if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||
rows, err := inv.Pending(ctx, time.Unix(0, 0))
|
||||
later := time.Now().Add(inventory.KeptFor + time.Hour)
|
||||
settlePending(ctx, open, later)
|
||||
rows, err := inv.Pending(ctx, later.Add(-inventory.KeptFor))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
if !r.Open() && (r.Raised == nil || r.Cleared != nil) {
|
||||
t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said)
|
||||
t.Fatalf("an ended pending assignment with no open condition is listed past %s: %+v", inventory.KeptFor, r)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") {
|
||||
t.Fatalf("the applied row was not deleted: %v", said)
|
||||
for _, r := range rows {
|
||||
if r.Module == "heard" {
|
||||
t.Fatalf("an applied wait is listed past %s: %+v", inventory.KeptFor, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -637,8 +640,9 @@ func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) {
|
||||
}
|
||||
settlePending(ctx, open, time.Now())
|
||||
settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
|
||||
if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] {
|
||||
t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err)
|
||||
// No key of the chain is removed (ADR 0408 §4): a cleared wait stays in the value of its key, saying it was cleared.
|
||||
if known, err := inv.PendingKnown(ctx, []string{row.ID}); err != nil || !known[row.ID] || rowOf(t, open, row.ID).Cleared == nil {
|
||||
t.Fatalf("a cleared row is not kept, saying so: %v", err)
|
||||
}
|
||||
|
||||
// On anchor, then anchor removed: the row goes, and its condition with it at the next tick.
|
||||
|
||||
@@ -23,7 +23,7 @@ func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool {
|
||||
return found
|
||||
}
|
||||
|
||||
func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
|
||||
func rowOf(t *testing.T, open *stores, id string) inventory.PendingAssignment {
|
||||
t.Helper()
|
||||
rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0))
|
||||
if err != nil {
|
||||
@@ -34,7 +34,7 @@ func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
|
||||
return r
|
||||
}
|
||||
}
|
||||
t.Fatalf("pending assignment %d is not on record", id)
|
||||
t.Fatalf("pending assignment %s is not on record", id)
|
||||
return inventory.PendingAssignment{}
|
||||
}
|
||||
|
||||
|
||||
@@ -610,6 +610,19 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
||||
"did not answer in ten minutes, so nothing of the delivery is built or sent.",
|
||||
Resolved: "Resolved: the backup is done and the delivery goes on"}
|
||||
}),
|
||||
kindStuck: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Something in a delivery has stopped moving",
|
||||
Explanation: "The watchman noticed a part of the delivery that did not move when it should have: a build " +
|
||||
"asked and not answered, a check that stopped, or a part that went offline. Nothing is lost, and it is " +
|
||||
"said again until it moves.",
|
||||
Resolved: "Resolved: it moves again"}
|
||||
}),
|
||||
kindSendKeyRefused: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "A delivery cannot watch a machine for its report",
|
||||
Explanation: "The bus refused the note that makes the mesh notice a machine that never reports on a send. " +
|
||||
"The delivery goes on, and the note is written again at its next look.",
|
||||
Resolved: "Resolved: the delivery watches the machine for its report again"}
|
||||
}),
|
||||
kindConsumerBehind: worded(func(o conditions.Observation) words {
|
||||
return words{Headline: "Messages pile up for a listener",
|
||||
Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " +
|
||||
|
||||
+89
-22
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/licences"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -500,33 +501,75 @@ func declarationWith(ctx context.Context, open *stores, node string,
|
||||
// And which build of each module it carries, for the send to record (novox/hq issue 259, ADR
|
||||
// 0221). Read only on the send path: a question about what would be sent records nothing.
|
||||
if choosing == Allocating {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
carried, err := carriedOn(ctx, open, node, plan, composed.LeftOut, true)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
before, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
if !known {
|
||||
before = nil
|
||||
}
|
||||
names := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
out.Builds = carriedBuilds(names, composed.LeftOut, current, before)
|
||||
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0245).
|
||||
kept, err := recordedKept(ctx, open, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
for m, was := range kept {
|
||||
if _, carried := out.Builds[m]; carried {
|
||||
out.Builds[m] = was
|
||||
out.Builds = carried
|
||||
out.Bindings = boundToData(plan, composed.LeftOut)
|
||||
}
|
||||
// And the module of each resource and the build of each module, to a machine whose node-engine said it reads them
|
||||
// (novox/hq issue 1145) and to no other; the same whatever the composition is for, so what a question about the
|
||||
// send digests is what the send digests.
|
||||
reads, err := open.inventory.ReadsModules(ctx, record.ID)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
if reads {
|
||||
carried := out.Builds
|
||||
if choosing != Allocating {
|
||||
if carried, err = carriedOn(ctx, open, node, plan, composed.LeftOut, false); err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
}
|
||||
out.Bindings = boundToData(plan, composed.LeftOut)
|
||||
runs, err := open.inventory.AssignmentRuns(ctx, node)
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
builds, err := wireBuilds(carried, runs, func(module, commit string) (string, error) {
|
||||
return open.inventory.BuildOf(ctx, module, commit)
|
||||
})
|
||||
if err != nil {
|
||||
return sendable{}, err
|
||||
}
|
||||
out.ReadsModules, out.moduleOf, out.buildIDs = true, composed.Owner, builds
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// carriedOn is the build of each module a declaration for node carries (see carriedBuilds), the module kept at the
|
||||
// build a machine runs included (ADR 0245). Only a send (strict) fails for a kept module that cannot be read: a question
|
||||
// about it is answered without.
|
||||
func carriedOn(ctx context.Context, open *stores, node string, plan catalogue.Resolution, leftOut map[string]string,
|
||||
strict bool) (map[string]string, error) {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
before, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !known {
|
||||
before = nil
|
||||
}
|
||||
names := make([]string, 0, len(plan.Modules))
|
||||
for _, m := range plan.Modules {
|
||||
names = append(names, m.Module)
|
||||
}
|
||||
out := carriedBuilds(names, leftOut, current, before)
|
||||
// A recorded module kept at the build the machine runs is recorded as carrying that one (ADR 0245).
|
||||
kept, err := recordedKept(ctx, open, node)
|
||||
if err != nil {
|
||||
if strict {
|
||||
return nil, err
|
||||
}
|
||||
kept = nil
|
||||
}
|
||||
for m, was := range kept {
|
||||
if _, carried := out[m]; carried {
|
||||
out[m] = was
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -2109,3 +2152,27 @@ func sortedNames[V any](m map[string]V) []string {
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// wireBuilds is the build of each carried module that goes on the wire (novox/hq issue 1145): the build its assignment
|
||||
// names when that assignment is of the commit carried, else the build made from that commit; a module with neither
|
||||
// is not named.
|
||||
func wireBuilds(carried map[string]string, runs map[string]chain.Ref, lookup func(module, commit string) (string, error)) (map[string]string, error) {
|
||||
out := map[string]string{}
|
||||
for module, commit := range carried {
|
||||
if commit == "" {
|
||||
continue
|
||||
}
|
||||
if ref, ok := runs[module]; ok && ref.Commit == commit && ref.Build != "" {
|
||||
out[module] = ref.Build
|
||||
continue
|
||||
}
|
||||
build, err := lookup(module, commit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if build != "" {
|
||||
out[module] = build
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// heldPublish is a bus that holds a plan-moved publish until it is let go, and keeps the body.
|
||||
type heldPublish struct {
|
||||
link.Bus
|
||||
let chan struct{}
|
||||
body chan []byte
|
||||
}
|
||||
|
||||
func (h *heldPublish) PublishSeatEvent(_ context.Context, _, _ string, body []byte) error {
|
||||
<-h.let
|
||||
h.body <- body
|
||||
return nil
|
||||
}
|
||||
|
||||
func aWalkWithSteps() inventory.Plan {
|
||||
built := time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC)
|
||||
return inventory.Plan{ID: "plan-1", Repository: "novox/one", Commit: "abc", State: "walking", Tier: 1,
|
||||
Created: built, Updated: built, Revision: 4,
|
||||
Tiers: [][]string{{"alpha"}, {"beta", "gamma"}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"alpha": {State: "built", BuiltAt: &built, Commit: "abc", First: []string{"ace"}},
|
||||
"beta": {State: "asked", Why: "waiting"},
|
||||
"gamma": {},
|
||||
},
|
||||
Times: &inventory.PlanTimes{Started: &built}, WalkSaid: inventory.WalkStarted}
|
||||
}
|
||||
|
||||
func TestAnAnnouncementIsThePlanAsItWasWhenSaidWhatTheWalkDoesNext(t *testing.T) { // 1260/AC1
|
||||
p := aWalkWithSteps()
|
||||
bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)}
|
||||
sayPlanMoved(t.Context(), bus, nil, p)
|
||||
|
||||
// The walk goes on, writing its steps while the announcement is being made.
|
||||
for i := 0; i < 2000; i++ {
|
||||
step := p.Modules["beta"]
|
||||
step.State, step.Why = "built", "changed after the call"
|
||||
step.First = append(step.First[:0], "novox")
|
||||
p.Modules["delta"] = &inventory.PlanModule{State: "asked"}
|
||||
p.Tiers[1][0] = "changed"
|
||||
p.Times.Started = nil
|
||||
delete(p.Modules, "delta")
|
||||
}
|
||||
close(bus.let)
|
||||
|
||||
var said inventory.Plan
|
||||
select {
|
||||
case body := <-bus.body:
|
||||
if err := json.Unmarshal(body, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("nothing was announced")
|
||||
}
|
||||
if said.Modules["beta"].State != "asked" || said.Modules["beta"].Why != "waiting" || said.Tiers[1][0] != "beta" ||
|
||||
len(said.Modules) != 3 || said.Times == nil || said.Times.Started == nil {
|
||||
t.Errorf("the announcement carries the walk as it went on, not as it was: %+v", said)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnnouncementOfAnUnchangedPlanIsTheBodyItWasBeforeTheCopy(t *testing.T) { // 1260/AC2
|
||||
p := aWalkWithSteps()
|
||||
bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)}
|
||||
close(bus.let)
|
||||
sayPlanMoved(t.Context(), bus, nil, p)
|
||||
var got []byte
|
||||
select {
|
||||
case got = <-bus.body:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("nothing was announced")
|
||||
}
|
||||
// What it was: the plan whole, its phases worked out, and what the walk says of itself.
|
||||
q := p
|
||||
q.Phases = q.WalkPhases(time.Now(), nil)
|
||||
want, err := json.Marshal(planMovedBody{Plan: q, walkSaid: sayWalk(t.Context(), nil, q)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var a, b map[string]any
|
||||
if json.Unmarshal(got, &a) != nil || json.Unmarshal(want, &b) != nil {
|
||||
t.Fatal("a body is not JSON")
|
||||
}
|
||||
delete(a, "phases") // worked out from the time of the call
|
||||
delete(b, "phases")
|
||||
if !reflect.DeepEqual(a, b) {
|
||||
t.Errorf("the body changed:\n got %s\nwant %s", got, want)
|
||||
}
|
||||
if a["walk"] != "plan-1" {
|
||||
t.Errorf("the walk's own words are not in the body: %s", got)
|
||||
}
|
||||
}
|
||||
@@ -400,33 +400,39 @@ func syncSilentKeys(ctx context.Context, open *stores) error {
|
||||
}
|
||||
|
||||
// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys
|
||||
// cannot be read, one cannot be put — the wait is looked at by a timer of its own, once, and the error is given back so
|
||||
// the look is retried.
|
||||
// cannot be read, one cannot be put — the refusal is raised as a condition naming the plan and the machine, never
|
||||
// stood in for by a timer, and the error is given back so the look is retried and writes the key again (novox/hq ADR
|
||||
// 0433 §4). The set of refusals is what this look found: a key written at the next look is no longer in it.
|
||||
func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error {
|
||||
byTimer := func() {
|
||||
for w, sent := range want {
|
||||
silentByTimer(w, sent, silentTTL(sent, now))
|
||||
refused := func(err error) map[silentWait]error {
|
||||
out := map[silentWait]error{}
|
||||
for w := range want {
|
||||
out[w] = err
|
||||
}
|
||||
return out
|
||||
}
|
||||
if c == nil {
|
||||
byTimer()
|
||||
noteRefusedSendKeys(refused(errors.New("the bus's clock for the waits is not there")))
|
||||
return nil
|
||||
}
|
||||
c = boundedClock{c}
|
||||
held, err := c.Held(ctx)
|
||||
if err != nil {
|
||||
byTimer()
|
||||
noteRefusedSendKeys(refused(err))
|
||||
return err
|
||||
}
|
||||
arm, drop := silentDiff(want, held)
|
||||
var first error
|
||||
notTaken := map[silentWait]error{}
|
||||
for _, w := range arm {
|
||||
if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil {
|
||||
silentByTimer(w, want[w], silentTTL(want[w], now))
|
||||
notTaken[w] = err
|
||||
if first == nil {
|
||||
first = err
|
||||
}
|
||||
}
|
||||
}
|
||||
noteRefusedSendKeys(notTaken)
|
||||
for _, w := range drop {
|
||||
if err := c.Reported(ctx, w); err != nil && first == nil {
|
||||
first = err
|
||||
@@ -435,29 +441,49 @@ func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time
|
||||
return first
|
||||
}
|
||||
|
||||
var silentTimers = struct {
|
||||
mu sync.Mutex
|
||||
timers map[silentWait]time.Time
|
||||
}{timers: map[silentWait]time.Time{}}
|
||||
// silentCallBound is how long one call to the silent clock may take while the look holds the plans (novox/hq issue 1121).
|
||||
// The vendored key listing reads its watcher without the context, so a consumer lost when the broker reloaded its user
|
||||
// list left the look waiting for good, with the plans and every node held.
|
||||
var silentCallBound = 20 * time.Second
|
||||
|
||||
// silentByTimer looks at the plans when a wait's time is up, for a process whose bus will not take the key
|
||||
// (DegradableGrants says why that can be): never silently never. Once for each send.
|
||||
func silentByTimer(w silentWait, sent time.Time, ttl time.Duration) {
|
||||
silentTimers.mu.Lock()
|
||||
defer silentTimers.mu.Unlock()
|
||||
if at, ok := silentTimers.timers[w]; ok && at.Equal(sent) {
|
||||
return
|
||||
// boundedClock is a clock whose every call is answered within silentCallBound, or fails: a call that never returns is
|
||||
// left behind on its own goroutine.
|
||||
type boundedClock struct{ silentClock }
|
||||
|
||||
func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, silentCallBound)
|
||||
defer cancel()
|
||||
type answer struct {
|
||||
v T
|
||||
err error
|
||||
}
|
||||
silentTimers.timers[w] = sent
|
||||
// retries: a rollout's wait for a machine, where the bus would not take its key. One per send, once.
|
||||
time.AfterFunc(ttl, func() {
|
||||
silentTimers.mu.Lock()
|
||||
if silentTimers.timers[w].Equal(sent) {
|
||||
delete(silentTimers.timers, w)
|
||||
}
|
||||
silentTimers.mu.Unlock()
|
||||
askToLook()
|
||||
done := make(chan answer, 1)
|
||||
go func() { v, err := call(ctx); done <- answer{v, err} }()
|
||||
select {
|
||||
case a := <-done:
|
||||
return a.v, a.err
|
||||
case <-ctx.Done():
|
||||
var zero T
|
||||
return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func (b boundedClock) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error {
|
||||
_, err := boundedCall(ctx, "arming a wait", func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, b.silentClock.Arm(ctx, w, ttl, sent)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (b boundedClock) Held(ctx context.Context) (map[silentWait]time.Time, error) {
|
||||
return boundedCall(ctx, "listing the waits", b.silentClock.Held)
|
||||
}
|
||||
|
||||
func (b boundedClock) Reported(ctx context.Context, w silentWait) error {
|
||||
_, err := boundedCall(ctx, "dropping a wait", func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, b.silentClock.Reported(ctx, w)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the
|
||||
|
||||
@@ -3,10 +3,12 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
@@ -159,23 +161,105 @@ func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
||||
return nil, errors.New("the bus is down")
|
||||
}
|
||||
|
||||
// A bus that will not say which keys it holds does not leave the waits unwatched: each is looked at by a timer of its own
|
||||
// and the error is given back, so the look is marked failed and retried (issue 974, review 1).
|
||||
func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) {
|
||||
// 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by
|
||||
// nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error
|
||||
// is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4).
|
||||
func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1
|
||||
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
t.Cleanup(func() { silentTimers.mu.Lock(); delete(silentTimers.timers, w); silentTimers.mu.Unlock() })
|
||||
err := keepSilentKeys(t.Context(), brokenClock{}, map[silentWait]time.Time{w: sent}, sent)
|
||||
if err == nil {
|
||||
t.Fatal("no error given back")
|
||||
for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{},
|
||||
"no clock": nil} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
noteRefusedSendKeys(nil)
|
||||
t.Cleanup(func() { noteRefusedSendKeys(nil) })
|
||||
err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent)
|
||||
if clock != nil && err == nil {
|
||||
t.Fatal("no error given back")
|
||||
}
|
||||
got := observeRefusedSendKeys()
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("observations %+v", got)
|
||||
}
|
||||
o := got[0]
|
||||
if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused ||
|
||||
!strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") {
|
||||
t.Fatalf("the condition does not name the plan and the machine: %+v", o)
|
||||
}
|
||||
})
|
||||
}
|
||||
silentTimers.mu.Lock()
|
||||
armed := silentTimers.timers[w].Equal(sent)
|
||||
silentTimers.mu.Unlock()
|
||||
if !armed {
|
||||
t.Error("no timer of its own watches the wait")
|
||||
}
|
||||
|
||||
// 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is.
|
||||
func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2
|
||||
w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
noteRefusedSendKeys(nil)
|
||||
t.Cleanup(func() { noteRefusedSendKeys(nil) })
|
||||
want := map[silentWait]time.Time{w: sent}
|
||||
if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 {
|
||||
t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys())
|
||||
}
|
||||
clock := &keptClock{}
|
||||
if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !clock.armed[w].Equal(sent) {
|
||||
t.Fatalf("the next look did not write the key again: %v", clock.armed)
|
||||
}
|
||||
if got := observeRefusedSendKeys(); len(got) != 0 {
|
||||
t.Fatalf("the condition stays after the key was written: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
type armRefusingClock struct{ silentClock }
|
||||
|
||||
func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
|
||||
func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error {
|
||||
return errors.New("the bus refuses the key")
|
||||
}
|
||||
|
||||
// keptClock is a clock that takes every key.
|
||||
type keptClock struct {
|
||||
silentClock
|
||||
armed map[silentWait]time.Time
|
||||
}
|
||||
|
||||
func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil }
|
||||
func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error {
|
||||
if c.armed == nil {
|
||||
c.armed = map[silentWait]time.Time{}
|
||||
}
|
||||
c.armed[w] = sent
|
||||
return nil
|
||||
}
|
||||
|
||||
// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is
|
||||
// looked at: the controller follows its own plan-moved (issue 974, round 1).
|
||||
|
||||
// A clock whose bus answers nothing, ever: a consumer lost when the broker reloaded its user list, a key listing that
|
||||
// never reaches its end.
|
||||
type hungClock struct{ silentClock }
|
||||
|
||||
func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) {
|
||||
select {} // never answers, whatever the context says (the vendored Keys reads its watcher without it)
|
||||
}
|
||||
|
||||
// issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans
|
||||
// and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition.
|
||||
func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) {
|
||||
was := silentCallBound
|
||||
silentCallBound = 50 * time.Millisecond
|
||||
t.Cleanup(func() { silentCallBound = was })
|
||||
w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"}
|
||||
sent := time.Now().UTC()
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("a clock that never answered was reported as fine")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the look is still waiting for a clock that never answers")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,9 +12,9 @@ import (
|
||||
)
|
||||
|
||||
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
|
||||
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there
|
||||
// after the commit; so no statement that writes the plan table may stand anywhere else, and writePlan, the one that
|
||||
// does, may be called from nowhere but SavePlansMoving. Checked by reading the sources, no store needed.
|
||||
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the
|
||||
// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts
|
||||
// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed.
|
||||
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
|
||||
var found []string
|
||||
@@ -31,11 +31,13 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
return err
|
||||
}
|
||||
text := string(raw)
|
||||
if write.MatchString(text) && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
||||
if write.MatchString(text) {
|
||||
found = append(found, path+" writes the plan table")
|
||||
}
|
||||
if strings.Contains(text, "writePlan(") && filepath.ToSlash(path) != "../../internal/inventory/plans.go" {
|
||||
found = append(found, path+" calls writePlan")
|
||||
// A plan record is written by the bucket's raw create and update, which only the plans' own file calls.
|
||||
if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" &&
|
||||
strings.Contains(text, "chain.RolloutsOf") {
|
||||
found = append(found, path+" reaches the rollouts bucket")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
@@ -51,17 +53,26 @@ func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(raw)
|
||||
if n := len(write.FindAllString(text, -1)); n != 1 {
|
||||
t.Errorf("plans.go has %d statements that write the plan table; want the one in writePlan", n)
|
||||
}
|
||||
calls := regexp.MustCompile(`writePlan\(ctx, tx`).FindAllStringIndex(text, -1)
|
||||
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
|
||||
next := strings.Index(text[save:], "\nfunc ")
|
||||
if len(calls) != 1 || calls[0][0] < save || calls[0][0] > save+next {
|
||||
t.Errorf("writePlan is called %d times, and not only from SavePlansMoving", len(calls))
|
||||
if next < 0 {
|
||||
next = len(text) - save
|
||||
}
|
||||
if !strings.Contains(text[save:save+next], "PlanSaved(i, *p)") {
|
||||
t.Error("SavePlansMoving does not say each plan it wrote")
|
||||
if !strings.Contains(text[save:save+next], "chainSavePlans(") {
|
||||
t.Error("SavePlansMoving does not write through chainSavePlans")
|
||||
}
|
||||
chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chainText := string(chainRaw)
|
||||
at := strings.Index(chainText, "func (i *Inventory) chainSavePlans")
|
||||
end := strings.Index(chainText[at:], "\nfunc ")
|
||||
if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") {
|
||||
t.Error("chainSavePlans does not say each plan it wrote")
|
||||
}
|
||||
if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 {
|
||||
t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -155,7 +155,7 @@ func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bo
|
||||
}
|
||||
|
||||
// retryRefusal is why a plan cannot be retried, or nothing.
|
||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error {
|
||||
switch {
|
||||
case p.State == inventory.PlanDone:
|
||||
return fmt.Errorf("%s is done; there is nothing to retry", p.ID)
|
||||
@@ -170,7 +170,7 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+
|
||||
"what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID)
|
||||
}
|
||||
return oneWalkAtATime(p, plans)
|
||||
return oneWalkAtATime(p, plans, onPath)
|
||||
}
|
||||
stopped := stoppedRollouts(p)
|
||||
if len(stopped) == 0 {
|
||||
@@ -194,20 +194,20 @@ func retryRefusal(p inventory.Plan, plans []inventory.Plan) error {
|
||||
"the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID)
|
||||
}
|
||||
}
|
||||
return oneWalkAtATime(p, plans)
|
||||
return oneWalkAtATime(p, plans, onPath)
|
||||
}
|
||||
|
||||
// oneWalkAtATime refuses a retry while an open walk, started or waiting for its word, is related to it (novox/hq
|
||||
// ADR 0276, ADR 0407 decision 4, issue 973): a walk retried beside a related one would be two walks on the same
|
||||
// things at once. A walk related to none may be retried beside them. The retried walk is judged by the modules it
|
||||
// plans, which already hold their dependents, and the repositories it carries.
|
||||
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan) error {
|
||||
func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error {
|
||||
mine := footprintOfWalk(p, nil)
|
||||
for _, q := range plans {
|
||||
if q.ID == p.ID || !q.Open() || q.Release != nil {
|
||||
continue
|
||||
}
|
||||
if ownWalk(p) || ownWalk(q) {
|
||||
if ownWalk(p, onPath) || ownWalk(q, onPath) {
|
||||
return fmt.Errorf("%s is open (%s) and one of them is on the controller's own path: one walk at a time — "+
|
||||
"retry %s once it ended", q.ID, q.Named(), p.ID)
|
||||
}
|
||||
@@ -327,7 +327,11 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) {
|
||||
}
|
||||
failed = toRetry(&p, recorded, byID)
|
||||
}
|
||||
if err := retryRefusal(p, plans); err != nil {
|
||||
onPath, err := ownPathNow(ctx, inv)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := retryRefusal(p, plans, onPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 {
|
||||
|
||||
@@ -629,7 +629,7 @@ func heardMachines(d *doctor) map[string]bool {
|
||||
return out
|
||||
}
|
||||
for _, m := range f.machines {
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) <= heartbeatBound(m.every) && !m.asleep() {
|
||||
if !m.lastHeard.IsZero() && !m.silent(f.now) && !m.asleep() {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// review round 3, fault B (1181): the test asserted a literal url, while issueOnTheNewBus works out the
|
||||
// machine's reachable address from the catalogue's network records (brokerReachableAt), not from this
|
||||
// process's environment — the real try found "nats://anchor.internal:5671", not the literal guessed here.
|
||||
// Fixed by calling issueProcess directly, with the known broker and reachable address given to it, exactly
|
||||
// as issued_to_vault_test.go's issue/issueWith and issueProvisionerClient are called directly; the url
|
||||
// asserted below is one of those two inputs, never a separate guess.
|
||||
//
|
||||
// review round 1, findings 3 and 4 (1181/AC1, AC2): module issue mints and delivers a process's own
|
||||
// account against the store — no bus to dial, busAddress "" — for mesh-mail's real manifest shape (review
|
||||
// round 1 finding 1: no own secret `broker` at all, its tools served by the tool runner), and issuing it
|
||||
// again is the path every bus account takes (`rollout mint`'s KindProcess case, and a re-run of `module
|
||||
// issue`): it rotates the password, and the old hash no longer matches.
|
||||
func TestModuleIssueMintsAndDeliversAProcesssAccount(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
theVaultIsInstalled(t, open)
|
||||
|
||||
m := mailSenderWithProcess()
|
||||
m.Moved = true // so handIssuedToVault's accountIsCovered is true, and the hand-off is exercised whole
|
||||
register(t, open, m)
|
||||
|
||||
vault := newStandInVault(t)
|
||||
vaultAskerForIssue = func(context.Context) vaultAsker { return vault.ask }
|
||||
t.Cleanup(func() { vaultAskerForIssue = func(context.Context) vaultAsker { return newStandInVault(nil).ask } })
|
||||
|
||||
pr, found := processPrincipalFor(m, "send")
|
||||
if !found {
|
||||
t.Fatal("mesh-mail declares no process send")
|
||||
}
|
||||
known := broker.Broker{Fingerprint: "ab"}
|
||||
const reachable = "10.0.0.1:4222"
|
||||
issue := func() {
|
||||
t.Helper()
|
||||
if err := issueProcess(ctx, inv, m, pr, "laptop", "", known, reachable); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
issue()
|
||||
user := broker.Principal{Kind: broker.KindProcess, Node: "laptop", Module: "mesh-mail", Process: "send"}.Username()
|
||||
if user != "laptop.mesh-mail.send" {
|
||||
t.Fatalf("the process's username: %q", user)
|
||||
}
|
||||
hash1, held, err := inv.BusUserHash(ctx, user)
|
||||
if err != nil || !held || hash1 == "" {
|
||||
t.Fatalf("the process's account was not minted: held %v, %v", held, err)
|
||||
}
|
||||
|
||||
// 1181/AC1: accepted with the provisioner client's JSON, and handed to the vault whole — the url and
|
||||
// the fingerprint being exactly what issueProcess was given above, not a guess at what it computes.
|
||||
taken := vault.taken["accept"]
|
||||
for _, want := range []string{`"url":"nats://` + reachable + `"`, `"fingerprint":"ab"`, `"node":"laptop"`,
|
||||
`"module":"mesh-mail"`, `"user":"laptop.mesh-mail.send"`, `"password"`} {
|
||||
if !strings.Contains(taken, want) {
|
||||
t.Errorf("the vault was handed %q, without %s", taken, want)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(vault.said(), `"name":"broker-send"`) {
|
||||
t.Errorf("the vault was not told the secret's name broker-send: %s", vault.said())
|
||||
}
|
||||
// The file's own owner (mesh-mail-send, review round 1 finding 7's area) is 1170's mechanism, proven
|
||||
// generically there and for this fixture in TestAProcessesOwnSecretIsOwnedByItsOwnAccount.
|
||||
|
||||
// 1181/AC2: issued again — the path `rollout mint`'s KindProcess case and a re-run of `module issue`
|
||||
// both take — rotates the password; the old hash no longer matches what is recorded.
|
||||
issue()
|
||||
hash2, held, err := inv.BusUserHash(ctx, user)
|
||||
if err != nil || !held {
|
||||
t.Fatalf("the process's account is gone after a re-issue: held %v, %v", held, err)
|
||||
}
|
||||
if hash1 == hash2 {
|
||||
t.Fatal("issuing the process's account again did not rotate its password")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
)
|
||||
|
||||
// mailSenderWithProcess is mesh-mail's real manifest shape (novox/hq ADR 0436 §7, §8, the lead's fixture):
|
||||
// no own secret named `broker` at all — its tools run in the machine's tool runner — and the process "send"
|
||||
// declares bus, whose own secret broker-send is where module issue delivers its account (ADR 0436 §2),
|
||||
// owned by mesh-mail-send, the user resource the process declares, at mode 0600 as every own secret is.
|
||||
func mailSenderWithProcess() catalogue.Manifest {
|
||||
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1","emits":["mail.send.sent"],
|
||||
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},
|
||||
{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",
|
||||
"bus":{"emits":["mail.send.sent"]}}]}`))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// review round 1, finding 1 (1181): a module declaring no `broker` of its own at all — mesh-mail's real
|
||||
// shape, tools served by the tool runner — still passes mayIssue on its process's bus declaration alone.
|
||||
func TestAModuleWithNoBrokerAndOnlyAProcessMayBeIssued(t *testing.T) {
|
||||
if err := mayIssue(mailSenderWithProcess()); err != nil {
|
||||
t.Fatalf("a module with no broker of its own, whose process declares bus, was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1181/AC1: a process with a bus declaration and no own secret broker-<process id> cannot be issued — the
|
||||
// same refusal mayIssue already gives a module with no `broker` (novox/hq ADR 0436 §2).
|
||||
func TestAProcessWithNoBrokerSecretCannotBeIssued(t *testing.T) {
|
||||
m := mailSenderWithProcess()
|
||||
delete(m.OwnSecrets, "broker-send")
|
||||
err := mayIssue(m)
|
||||
if err == nil {
|
||||
t.Fatal("a process with no broker-<id> own secret was issued an account")
|
||||
}
|
||||
for _, want := range []string{"mesh-mail", "send", "broker-send", "own-secrets"} {
|
||||
if !strings.Contains(err.Error(), want) {
|
||||
t.Errorf("the refusal does not say %q: %v", want, err)
|
||||
}
|
||||
}
|
||||
if err := mayIssue(mailSenderWithProcess()); err != nil {
|
||||
t.Errorf("a process declaring its own broker-<id> secret was refused: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 1181/AC1: the account held for a process is the provisioner client's form — {url, fingerprint, node,
|
||||
// module, user, password} — named for the process, never the module's own account.
|
||||
func TestAProcesssHeldAccountIsTheProvisionersForm(t *testing.T) {
|
||||
user := broker.Principal{Kind: broker.KindProcess, Node: "novox", Module: "mesh-mail", Process: "send"}.Username()
|
||||
if user != "novox.mesh-mail.send" {
|
||||
t.Fatalf("the process's username: %q", user)
|
||||
}
|
||||
held, err := heldProcessAccount(broker.Broker{Fingerprint: "abc"}, "novox.example:4222", "novox", "mesh-mail",
|
||||
user, "a-password")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(held, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]any{"url": "nats://novox.example:4222", "fingerprint": "abc", "node": "novox",
|
||||
"module": "mesh-mail", "user": "novox.mesh-mail.send", "password": "a-password"}
|
||||
for k, v := range want {
|
||||
if got[k] != v {
|
||||
t.Errorf("%s: %v, want %v", k, got[k], v)
|
||||
}
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Errorf("the held account carries %v; want exactly %v (never claims — a process holds no seat)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// 1181/AC1: composed, the process's own secret belongs to its own account at 0600 (novox/hq ADR 0436 §1, §2;
|
||||
// the owner mechanism is issue 1170's, proven generically there — this is mesh-mail's own shape).
|
||||
func TestAProcessesOwnSecretIsOwnedByItsOwnAccount(t *testing.T) {
|
||||
m := mailSenderWithProcess()
|
||||
r := catalogue.Resolution{Node: "novox", Modules: []catalogue.Manifest{m}}
|
||||
out, err := r.Declaration(catalogue.Rendering{
|
||||
Needed: map[string]map[string]string{"mesh-mail": {"broker-send": "sealed-account"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("composing the declaration: %v", err)
|
||||
}
|
||||
var found bool
|
||||
for _, res := range out {
|
||||
if res["type"] == "file" && res["path"] == "/var/lib/mesh-mail/broker-send" {
|
||||
found = true
|
||||
if got := res["owner"]; got != "mesh-mail-send" {
|
||||
t.Fatalf("broker-send's owner: %v; want mesh-mail-send", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("broker-send was not composed at all")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// mailSenderLiveManifest is mesh-mail's future manifest shape whole (novox/hq ADR 0436 §7, §8): the
|
||||
// process "send" declares every clause of bus (emits, its one work-queue stream, its state, asks, and its
|
||||
// invoked seat verb), so the account module issue delivers for it can be put through every grant at once.
|
||||
func mailSenderLiveManifest(t *testing.T) catalogue.Manifest {
|
||||
t.Helper()
|
||||
m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-mail","version":"1",
|
||||
"emits":["mail.send.sent","mail.send.failed"],"state":["sends"],
|
||||
"streams":[{"name":"mail-sends","events":["mail.send.approved"],"max-age-days":1,"max-mib":8,
|
||||
"retention":"work-queue","max-deliveries":5}],
|
||||
"own-secrets":{"broker-send":{"path":"/var/lib/mesh-mail/broker-send","owner":"mesh-mail-send"}},
|
||||
"resources":[{"id":"account","type":"user","name":"mesh-mail-send"},
|
||||
{"id":"send","type":"process","run":["./mesh-mail","send"],"user":"mesh-mail-send",
|
||||
"bus":{"emits":["mail.send.sent","mail.send.failed"],"queues":["mail-sends"],"state":["sends"],
|
||||
"asks":true,"invokes":["seat:operator-channel.tell"]}}]}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// operatorChannelWithTell is operator-channel as messenger declares it (broker.operatorChannel, novox/hq
|
||||
// ADR 0259 §3), with the one verb this release adds: `tell` (novox/hq ADR 0436 §1's own fixture), so the
|
||||
// process's invoked seat verb has a holder to answer it.
|
||||
func operatorChannelWithTell() broker.Seat {
|
||||
return broker.Seat{Name: "operator-channel", Scope: "mesh", Accepts: []string{"ask", "cancel"},
|
||||
Emits: []string{"decided"}, Serves: []string{"open", "history", "notify", "tell"},
|
||||
ByCaller: []string{"ask", "cancel", "decided"}, Records: []string{"messenger_asks"}}
|
||||
}
|
||||
|
||||
// review round 1, finding 5 (1181/AC3), the release's acceptance test, on a real bus and the store: the
|
||||
// account module issue delivers for mesh-mail's process "send" — composed from the manifest the real way
|
||||
// (inventory.ProcessesOf -> broker.Users -> broker.ComposeAccounts), minted with the password module issue
|
||||
// mints (aMesh) — does exactly what its bus declares and nothing else.
|
||||
func TestAProcessAccountOnARealBusAndStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
m := mailSenderLiveManifest(t)
|
||||
operatorChannel := operatorChannelWithTell()
|
||||
mailSends := broker.ModuleStream{Module: "mesh-mail", Name: "MAIL_SENDS", Events: []string{"mail.send.approved"},
|
||||
MaxAgeDays: 1, MaxMiB: 8, WorkQueue: true, Duplicates: 1}
|
||||
records := broker.Records{Nodes: []string{"laptop"}, Assigned: map[string][]broker.Declared{"laptop": {
|
||||
{Module: broker.RuntimeModule},
|
||||
{Module: "messenger", Holds: []broker.Seat{operatorChannel}, RunsAs: "messenger"},
|
||||
{Module: "mesh-mail", NoAccount: true, Uses: []broker.Seat{operatorChannel}, Streams: []broker.ModuleStream{mailSends},
|
||||
Processes: inventory.ProcessesOf(m)},
|
||||
}}}
|
||||
users, err := broker.Users(records)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The four principals this test drives, each minted the password module issue mints — the same
|
||||
// inv.MintBusPassword call issueProcess makes for the process account. mesh-mail publishes its own
|
||||
// stream's events as the tool runner carrying it (review round 2's finding): the controller's own
|
||||
// grants do not cover a module's stream, only the runtime that carries the module does.
|
||||
var admin, runtime, messenger, sender broker.Principal
|
||||
for _, p := range users {
|
||||
switch {
|
||||
case p.Kind == broker.KindController:
|
||||
admin = p
|
||||
case p.Kind == broker.KindNodeTools:
|
||||
runtime = p
|
||||
case p.Module == "messenger":
|
||||
messenger = p
|
||||
case p.Kind == broker.KindProcess:
|
||||
sender = p
|
||||
}
|
||||
}
|
||||
if admin.Kind == "" || runtime.Kind == "" || messenger.Kind == "" || sender.Kind == "" {
|
||||
t.Fatalf("not every principal was composed: admin=%+v runtime=%+v messenger=%+v sender=%+v",
|
||||
admin, runtime, messenger, sender)
|
||||
}
|
||||
if sender.Username() != "laptop.mesh-mail.send" {
|
||||
t.Fatalf("the process's username: %q", sender.Username())
|
||||
}
|
||||
|
||||
passwords := map[string]string{}
|
||||
hashes := map[string]string{}
|
||||
for _, p := range []struct {
|
||||
principal broker.Principal
|
||||
kind string
|
||||
}{{admin, inventory.BusController}, {runtime, inventory.BusNodeTools}, {messenger, inventory.BusModule},
|
||||
{sender, inventory.BusProcess}} {
|
||||
pw, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: p.principal.Username(), Kind: p.kind,
|
||||
Node: p.principal.Node, Module: p.principal.Module})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
passwords[p.principal.Username()] = pw
|
||||
hash, held, err := inv.BusUserHash(ctx, p.principal.Username())
|
||||
if err != nil || !held {
|
||||
t.Fatalf("the minted password of %s was not recorded: %v, %v", p.principal.Username(), held, err)
|
||||
}
|
||||
hashes[p.principal.Username()] = hash
|
||||
}
|
||||
filled, missing := broker.WithPasswords([]broker.Principal{admin, runtime, messenger, sender}, hashes)
|
||||
if len(missing) > 0 {
|
||||
t.Fatalf("no password for %v", missing)
|
||||
}
|
||||
|
||||
accounts, err := broker.ComposeAccounts(filled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conf := filepath.Join(t.TempDir(), "bus.conf")
|
||||
if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+
|
||||
`"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
opts, err := server.ProcessConfigFile(conf)
|
||||
if err != nil {
|
||||
t.Fatalf("the composed accounts do not parse: %v", err)
|
||||
}
|
||||
opts.NoLog, opts.NoSigs = true, true
|
||||
s, err := server.NewServer(opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
go s.Start()
|
||||
if !s.ReadyForConnections(10 * time.Second) {
|
||||
t.Fatal("the bus did not come up")
|
||||
}
|
||||
defer s.Shutdown()
|
||||
|
||||
// Every connection names its own inbox prefix: the grant is `_INBOX.<username>.>`, and the client
|
||||
// library's own default random inbox matches nothing granted, so replies are refused without it.
|
||||
adminConn, err := broker.Dial(s.ClientURL(), nats.UserInfo(admin.Username(), passwords[admin.Username()]),
|
||||
nats.CustomInboxPrefix("_INBOX."+admin.Username()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer adminConn.Close()
|
||||
if err := adminConn.EnsureStream(mailSends.Stream()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := adminConn.EnsureConsumer(broker.QueueConsumer(sender, sender.Queues[0])); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := adminConn.EnsureBucket(broker.Bucket{Module: "mesh-mail", Name: "sends"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
js, err := jetstream.New(adminConn.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "messenger_asks"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The machine's tool runner, which carries mesh-mail (it has no account of its own) and so is the
|
||||
// principal that publishes its own stream's events on its behalf.
|
||||
runtimeConn, err := nats.Connect(s.ClientURL(), nats.UserInfo(runtime.Username(), passwords[runtime.Username()]),
|
||||
nats.CustomInboxPrefix("_INBOX."+runtime.Username()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer runtimeConn.Close()
|
||||
|
||||
// messenger, the router, answers exactly its granted verbs — never a wildcard the grant does not give it.
|
||||
holder, err := nats.Connect(s.ClientURL(), nats.UserInfo(messenger.Username(), passwords[messenger.Username()]),
|
||||
nats.CustomInboxPrefix("_INBOX."+messenger.Username()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Close()
|
||||
heard := make(chan string, 1)
|
||||
for _, verb := range []string{"open", "history", "notify", "tell"} {
|
||||
if _, err := holder.Subscribe("mesh.seat.operator-channel.tool."+verb, func(m *nats.Msg) {
|
||||
heard <- m.Subject
|
||||
_ = m.Respond([]byte(`{"result":{}}`))
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
_ = holder.Flush()
|
||||
|
||||
refusals := make(chan error, 8)
|
||||
sender2, err := nats.Connect(s.ClientURL(), nats.UserInfo(sender.Username(), passwords[sender.Username()]),
|
||||
nats.CustomInboxPrefix("_INBOX."+sender.Username()),
|
||||
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err }))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer sender2.Close()
|
||||
|
||||
// 1. Its event.
|
||||
if err := sender2.Publish("mesh.mod.mesh-mail.event.mail.send.sent", []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 2. Its state: writes and reads its own bucket.
|
||||
senderKV, err := jetstream.New(sender2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucket, err := senderKV.KeyValue(ctx, "mesh-mail_sends")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := bucket.Put(ctx, "key", []byte("value")); err != nil {
|
||||
t.Fatalf("the process may not write its own state: %v", err)
|
||||
}
|
||||
if got, err := bucket.Get(ctx, "key"); err != nil || string(got.Value()) != "value" {
|
||||
t.Fatalf("the process may not read its own state back: %v, %v", got, err)
|
||||
}
|
||||
|
||||
// 3. Its queue: pulls and acknowledges MAIL_SENDS through its own consumer. Published as the tool
|
||||
// runner carrying mesh-mail — the live path a module's own stream's event takes — never as the
|
||||
// controller, whose own grants do not cover a module's stream at all.
|
||||
runtimeJS, err := jetstream.New(runtimeConn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtimeJS.Publish(ctx, broker.ModuleStreamSubject("mesh-mail", "mail.send.approved"), []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reading, err := jetstream.New(sender2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer, err := reading.Consumer(ctx, "MAIL_SENDS", broker.QueueConsumer(sender, sender.Queues[0]).Name)
|
||||
if err != nil {
|
||||
t.Fatalf("the process may not bind its own queue consumer: %v", err)
|
||||
}
|
||||
batch, err := consumer.Fetch(1, jetstream.FetchMaxWait(3*time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := false
|
||||
for msg := range batch.Messages() {
|
||||
got = true
|
||||
if err := msg.Ack(); err != nil {
|
||||
t.Fatalf("the process may not acknowledge its own queue: %v", err)
|
||||
}
|
||||
}
|
||||
if !got {
|
||||
t.Fatal("the process did not pull the message its own queue holds")
|
||||
}
|
||||
|
||||
// 4. Its invoked seat verb, in its own name.
|
||||
if _, err := sender2.Request("mesh.seat.operator-channel.tool.tell", []byte(`{}`), 3*time.Second); err != nil {
|
||||
t.Fatalf("the process may not call operator-channel.tell: %v", err)
|
||||
}
|
||||
select {
|
||||
case s := <-heard:
|
||||
if s != "mesh.seat.operator-channel.tool.tell" {
|
||||
t.Fatalf("messenger heard %s", s)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("messenger never heard the call")
|
||||
}
|
||||
|
||||
// 5. Refused, one at a time, so each refusal is counted against the attempt that caused it rather than
|
||||
// any other: answering a tool call it holds no seat and serves no tool for.
|
||||
if _, err := sender2.Subscribe("mesh.mod.mesh-mail.tool.>", func(*nats.Msg) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = sender2.Flush()
|
||||
select {
|
||||
case err := <-refusals:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("subscribing a tool call was not refused as a permissions violation: %v", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("subscribing a tool call the process does not serve was not refused")
|
||||
}
|
||||
// 6. Refused: publishing a warrant — only the holder, messenger, may say operator-channel decided.
|
||||
if err := sender2.Publish("mesh.seat.operator-channel.event.decided.mesh-mail", []byte("{}")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = sender2.Flush()
|
||||
select {
|
||||
case err := <-refusals:
|
||||
if !errors.Is(err, nats.ErrPermissionViolation) {
|
||||
t.Errorf("publishing a warrant was not refused as a permissions violation: %v", err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("publishing a warrant the process does not hold the seat for was not refused")
|
||||
}
|
||||
}
|
||||
@@ -172,6 +172,11 @@ func serve(ctx context.Context) (err error) {
|
||||
return err
|
||||
}
|
||||
defer server.Close()
|
||||
// The chain on the bus, and the move of what the tables still hold into it (novox/hq ADR 0408 §8): before anything
|
||||
// reads builds, rollouts or assignments.
|
||||
if err := attachTheChain(ctx, inv, server); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The bus's own objects, asserted on every start. **Not created once at genesis**: a stream
|
||||
// somebody deleted, a mesh raised from a restored backup, or a bus whose data directory was
|
||||
@@ -204,7 +209,8 @@ func serve(ctx context.Context) (err error) {
|
||||
}
|
||||
// And the merges mesh-delivery made, kept a week with the ask each was made on: what registers a module that
|
||||
// moved between repositories on the operator's Approve (novox/hq ADR 0420 §7, issue 1025).
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream}); err != nil {
|
||||
if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: open.inventory.Catalogued,
|
||||
kept: following{open: open}.resumeFirstBuilds}); err != nil {
|
||||
return err
|
||||
}
|
||||
// No timer reads the merges back (novox/hq ADR 0407 decision 3, issue 973): the forge's event and the durable
|
||||
|
||||
@@ -68,7 +68,7 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
{plan("plan-o", inventory.PlanBuilding, at), nil, "still building"},
|
||||
{failed, []inventory.Plan{plan("plan-2", inventory.PlanRolling, at.Add(time.Hour))}, "plan-2 supersedes it"},
|
||||
} {
|
||||
err := retryRefusal(c.p, c.others)
|
||||
err := retryRefusal(c.p, c.others, ownPathOf(nil))
|
||||
if err == nil || !strings.Contains(err.Error(), c.says) {
|
||||
t.Errorf("%s: %v, wanted it to say %q", c.p.ID, err, c.says)
|
||||
}
|
||||
@@ -76,12 +76,12 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
stopped := failed
|
||||
stopped.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}}
|
||||
stopped.Note = "a stopped at its first machine"
|
||||
if err := retryRefusal(stopped, nil); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||
if err := retryRefusal(stopped, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 0") {
|
||||
t.Errorf("a plan with nothing failed to build was retried: %v", err)
|
||||
}
|
||||
// A failed or done plan does not supersede it.
|
||||
if err := retryRefusal(failed, []inventory.Plan{plan("plan-4", inventory.PlanFailed, at.Add(time.Hour)),
|
||||
plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}); err != nil {
|
||||
plan("plan-5", inventory.PlanDone, at.Add(time.Hour))}, ownPathOf(nil)); err != nil {
|
||||
t.Errorf("refused for a plan that does not supersede it: %v", err)
|
||||
}
|
||||
// Another branch's open walk, or an older one, does not supersede it, and is one walk open: one at a time
|
||||
@@ -89,7 +89,7 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) {
|
||||
other := plan("plan-3", inventory.PlanBuilding, at.Add(time.Hour))
|
||||
other.Branch = "release"
|
||||
for _, open := range []inventory.Plan{other, plan("plan-0", inventory.PlanBuilding, at.Add(-time.Hour))} {
|
||||
if err := retryRefusal(failed, []inventory.Plan{open}); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
|
||||
if err := retryRefusal(failed, []inventory.Plan{open}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") {
|
||||
t.Errorf("retried beside the open walk %s: %v", open.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,12 +2,15 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/recordmirror"
|
||||
)
|
||||
@@ -41,7 +44,6 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
||||
return e
|
||||
}
|
||||
sources := map[string]func(context.Context) (map[string][]byte, error){
|
||||
recordmirror.Assignments.Name: inv.AssignmentsMirror,
|
||||
recordmirror.DeclaredConfig.Name: inv.DeclaredConfigMirror,
|
||||
recordmirror.Predefined.Name: inv.PredefinedMirror,
|
||||
}
|
||||
@@ -49,8 +51,12 @@ func mirroringRecords(ctx context.Context, inv *inventory.Inventory, conn *nats.
|
||||
failing := map[string]string{}
|
||||
tick := time.NewTicker(mirrorEvery)
|
||||
defer tick.Stop()
|
||||
events := &assignmentEvents{}
|
||||
for {
|
||||
if holds() {
|
||||
// The assignments are keys of the bus, written by the verbs that change them (novox/hq ADR 0408): this tick
|
||||
// keeps the provisioner's plan in their values and says what changed. They are no mirror of the store.
|
||||
keepingAssignments(ctx, inv, announce, epoch, events, failing)
|
||||
for _, m := range recordmirror.Mirrors() {
|
||||
why := mirrorOnce(ctx, api, m, writers, sources[m.Name], announce, epoch, holds)
|
||||
if why != failing[m.Name] {
|
||||
@@ -104,3 +110,69 @@ func mirrorOnce(ctx context.Context, api jetstream.JetStream, m recordmirror.Mir
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// assignmentEvents is the assignments as the bucket held them at the last look, to say what changed since.
|
||||
type assignmentEvents struct {
|
||||
known map[string]inventory.KeptValue
|
||||
}
|
||||
|
||||
// keepingAssignments is the tick's work for the assignments once they are on the bus: the provisioner's plan written into
|
||||
// the values it belongs to, and one `assignment-changed` event for each key that is new, changed or ended since the last
|
||||
// look — whoever wrote it, a verb in this process or a command in another. False when they are not on the bus yet.
|
||||
func keepingAssignments(ctx context.Context, inv *inventory.Inventory, announce recordmirror.Announcer, epoch func() uint64,
|
||||
events *assignmentEvents, failing map[string]string) {
|
||||
reading, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
const name = "assignments"
|
||||
if _, _, err := inv.KeepAssignmentPlans(reading); err != nil {
|
||||
if failing[name] != err.Error() {
|
||||
failing[name] = err.Error()
|
||||
fmt.Printf("the provisioner's plans are not kept in the assignments: %v\n", err)
|
||||
}
|
||||
} else if failing[name] != "" {
|
||||
fmt.Printf("the provisioner's plans are kept in the assignments again\n")
|
||||
failing[name] = ""
|
||||
}
|
||||
now, err := inv.KeptAssignments(reading)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if events.known != nil {
|
||||
keys := make([]string, 0, len(now)+len(events.known))
|
||||
for key := range now {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
for key := range events.known {
|
||||
if _, still := now[key]; !still {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
was, had := events.known[key]
|
||||
is, has := now[key]
|
||||
e := recordmirror.Event{Mirror: recordmirror.Assignments.Name, Key: key, Revision: is.Revision, Epoch: epoch(),
|
||||
At: time.Now().UTC()}
|
||||
switch {
|
||||
case has && !had:
|
||||
e.Change = recordmirror.Added
|
||||
case has && had && was.Revision != is.Revision:
|
||||
e.Change, e.Fields = recordmirror.Changed, recordmirror.ChangedFields(was.Value, is.Value)
|
||||
case !has && had:
|
||||
e.Change, e.Revision = recordmirror.Removed, 0
|
||||
default:
|
||||
continue
|
||||
}
|
||||
body, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for try := 0; try < 2; try++ {
|
||||
if err = announce.PublishSeatEvent(reading, broker.ControllerSeat, recordmirror.Assignments.Event, body); err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
events.known = now
|
||||
}
|
||||
|
||||
@@ -144,12 +144,12 @@ func relatedOpen(cand footprint, candOwn bool, open []openWalk, edges []inventor
|
||||
|
||||
// ownWalk says a walk is on the controller's own path: its batch says so, or it plans a module of that path
|
||||
// (issue 973).
|
||||
func ownWalk(p inventory.Plan) bool {
|
||||
func ownWalk(p inventory.Plan, onPath map[string]string) bool {
|
||||
if p.OwnPath() {
|
||||
return true
|
||||
}
|
||||
for name := range p.Modules {
|
||||
if _, own := onTheControllersPath[name]; own {
|
||||
if _, own := onPath[name]; own {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -173,19 +173,19 @@ func startedWalks(plans []inventory.Plan, id string, created time.Time) []invent
|
||||
|
||||
// openWalkOf is an open walk as a change is judged against it: what it reaches, from its planned modules and the
|
||||
// merges the record holds for it.
|
||||
func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (openWalk, error) {
|
||||
func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan, onPath map[string]string) (openWalk, error) {
|
||||
kept, err := inv.MergesOf(ctx, p.ID)
|
||||
if err != nil {
|
||||
return openWalk{}, err
|
||||
}
|
||||
return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p)}, nil
|
||||
return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p, onPath)}, nil
|
||||
}
|
||||
|
||||
// openWalksOf are openWalkOf for each of plans.
|
||||
func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) ([]openWalk, error) {
|
||||
func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan, onPath map[string]string) ([]openWalk, error) {
|
||||
var out []openWalk
|
||||
for _, p := range plans {
|
||||
w, err := openWalkOf(ctx, inv, p)
|
||||
w, err := openWalkOf(ctx, inv, p, onPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -119,19 +119,19 @@ func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) {
|
||||
|
||||
// A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973).
|
||||
func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) {
|
||||
if ownWalk(walkOf("novox/one", "notes")) {
|
||||
if ownWalk(walkOf("novox/one", "notes"), ownPathOf(nil)) {
|
||||
t.Error("an ordinary walk read as on the controller's own path")
|
||||
}
|
||||
flagged := walkOf("novox/one", "notes")
|
||||
flagged.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||
if !ownWalk(flagged) {
|
||||
if !ownWalk(flagged, ownPathOf(nil)) {
|
||||
t.Error("a walk whose batch is own-path read as ordinary")
|
||||
}
|
||||
var module string
|
||||
for m := range onTheControllersPath {
|
||||
module = m
|
||||
}
|
||||
if !ownWalk(walkOf("novox/one", "notes", module)) {
|
||||
if !ownWalk(walkOf("novox/one", "notes", module), ownPathOf(nil)) {
|
||||
t.Errorf("a walk planning %s read as ordinary", module)
|
||||
}
|
||||
}
|
||||
@@ -168,15 +168,15 @@ func TestARetryWaitsOnlyForAnOpenWalkItIsRelatedTo(t *testing.T) {
|
||||
p.ID, p.State = id, inventory.PlanBuilding
|
||||
return p
|
||||
}
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err != nil {
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err != nil {
|
||||
t.Errorf("refused beside an unrelated walk: %v", err)
|
||||
}
|
||||
err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")})
|
||||
err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")}, ownPathOf(nil))
|
||||
if err == nil || !strings.Contains(err.Error(), "plan-3") || !strings.Contains(err.Error(), "both move notes") {
|
||||
t.Errorf("a retry beside a walk moving the same module: %v", err)
|
||||
}
|
||||
failed.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}}
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}); err == nil {
|
||||
if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err == nil {
|
||||
t.Error("an own-path retry went beside an open walk")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1589,7 +1589,9 @@ func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalog
|
||||
var other []string
|
||||
listID := ""
|
||||
for _, m := range plan.Modules {
|
||||
if _, was := sent[m.Module]; !was {
|
||||
// A module no build is recorded for is carried with no commit and names none on its key (issue 1109), so it is "sent"
|
||||
// only by the comparison of the summaries below; a module with a commit that was never sent is a new assignment.
|
||||
if _, was := sent[m.Module]; !was && declared.Builds[m.Module] != "" {
|
||||
other = append(other, m.Module+" newly assigned")
|
||||
}
|
||||
if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"github.com/novox/mesh-controller/internal/builder"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
@@ -249,6 +250,68 @@ func appliedEach(module string, since func(node string) time.Time, running []str
|
||||
return len(waiting) == 0, waiting
|
||||
}
|
||||
|
||||
// gateNeed is a module the next tier waits for: the machines running it, and from when each must have reported.
|
||||
type gateNeed struct {
|
||||
module string
|
||||
running []string
|
||||
since func(node string) time.Time
|
||||
}
|
||||
|
||||
// silentNodes is the machines a walk sent and heard nothing from past ApplySilentAfter, on whichever tier it sent
|
||||
// them (novox/hq ADR 0360 §5, issue 1160): the one rule both the gates between tiers and the walk's end read, through
|
||||
// walkReports. A machine that reports on its send is not silent from then on.
|
||||
func silentNodes(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[string]bool {
|
||||
_, _, silent := walkReports(p, reports, now, prov)
|
||||
out := map[string]bool{}
|
||||
for _, n := range silent {
|
||||
out[n] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// withoutSilent is the machines of running that are not silent: a later tier is not sent to a machine the walk heard
|
||||
// nothing from, which keeps its want of the plan's builds for the send it gets when it is heard again (ADR 0360 §5).
|
||||
func withoutSilent(running []string, silent map[string]bool) []string {
|
||||
return slices.DeleteFunc(slices.Clone(running), func(n string) bool { return silent[n] })
|
||||
}
|
||||
|
||||
// settleTierGate is the wait between tiers on the reports of the machines running what the next tier needs: what is
|
||||
// still waited for ("module on a, b"), the silent machines left out of the wait, sorted, and whether every machine the
|
||||
// gate waits on is silent, so that nothing it needs was delivered anywhere (issue 1160).
|
||||
func settleTierGate(needs []gateNeed, reports []inventory.Reported, silent map[string]bool) (waiting, quiet []string, lost bool) {
|
||||
all, silentAll := 0, 0
|
||||
seenQuiet := map[string]bool{}
|
||||
seenNode := map[string]bool{}
|
||||
for _, need := range needs {
|
||||
for _, n := range need.running {
|
||||
if !seenNode[n] {
|
||||
seenNode[n] = true
|
||||
all++
|
||||
if silent[n] {
|
||||
silentAll++
|
||||
}
|
||||
}
|
||||
}
|
||||
_, on := appliedEach(need.module, need.since, need.running, reports)
|
||||
var still []string
|
||||
for _, n := range on {
|
||||
if silent[n] {
|
||||
if !seenQuiet[n] {
|
||||
seenQuiet[n] = true
|
||||
quiet = append(quiet, n)
|
||||
}
|
||||
continue
|
||||
}
|
||||
still = append(still, n)
|
||||
}
|
||||
if len(still) > 0 {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s", need.module, strings.Join(still, ", ")))
|
||||
}
|
||||
}
|
||||
sort.Strings(quiet)
|
||||
return waiting, quiet, all > 0 && silentAll == all
|
||||
}
|
||||
|
||||
func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) {
|
||||
at := map[string]*time.Time{}
|
||||
for _, r := range reports {
|
||||
@@ -638,11 +701,15 @@ func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Pl
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
open, err := openWalksOf(ctx, inv, started)
|
||||
onPath, err := ownPathNow(ctx, inv)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
mine, err := openWalkOf(ctx, inv, p)
|
||||
open, err := openWalksOf(ctx, inv, started, onPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
mine, err := openWalkOf(ctx, inv, p, onPath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
@@ -868,6 +935,22 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
// being behind. Now every module of the tier whose first machine is the same is sent there in one
|
||||
// send, judged there by one gate — kept on the first of them, the others gated by it — and the rest
|
||||
// of the tier's machines are sent once, together, when the gates have passed.
|
||||
// **A machine silent past the bound is left as it was, on every tier** (novox/hq ADR 0360 §5, issue 1160): the
|
||||
// walk's end names it, and the gates between tiers and the sends of the tiers after do not wait for it or go to it.
|
||||
allReports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
silent := silentNodes(*p, allReports, time.Now().UTC(), provLookup(ctx))
|
||||
// **A machine away when a tier is sent is not sent to and not waited for** (ADR 0435 §1, issue 1161): read once,
|
||||
// for a tier that has something left to send.
|
||||
var away map[string]bool
|
||||
for _, m := range tier {
|
||||
if s := p.Modules[m]; s != nil && s.SentAt == nil && s.FirstAt == nil && s.State != planDeleted {
|
||||
away = judgedAway(ctx, open, time.Now().UTC())
|
||||
break
|
||||
}
|
||||
}
|
||||
var pending []string
|
||||
firstOn := map[string][]string{}
|
||||
runningOf := map[string][]string{}
|
||||
@@ -902,6 +985,19 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if goes != nil {
|
||||
running = slices.DeleteFunc(running, func(n string) bool { return !goes[n] })
|
||||
}
|
||||
if state.FirstAt == nil {
|
||||
running = withoutSilent(running, silent)
|
||||
passed := withoutSilent(running, away)
|
||||
if len(passed) < len(running) {
|
||||
for _, n := range running {
|
||||
if away[n] && !slices.Contains(p.Away, n) {
|
||||
fmt.Printf("%s: %s is away; %s is not sent to it and not waited for\n", p.ID, n, m)
|
||||
}
|
||||
}
|
||||
rememberAway(p, running, away)
|
||||
running = passed
|
||||
}
|
||||
}
|
||||
runningOf[m] = running
|
||||
// **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source
|
||||
// of the build every machine running it was sent, it is registered with that build's artifacts —
|
||||
@@ -1054,7 +1150,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
}
|
||||
sort.Strings(machines)
|
||||
sent, err := sendRollout(withScope(ctx, scope), open, machines)
|
||||
was := map[string]map[string]chain.Ref{}
|
||||
for _, n := range machines {
|
||||
was[n] = ranBefore(ctx, inv, n)
|
||||
}
|
||||
sent, err := sendRollout(withScope(sendingAs(ctx, p), scope), open, machines)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("sending %s to %s after tier %d: %w", strings.Join(rest, ", "),
|
||||
strings.Join(machines, ", "), p.Tier, err)
|
||||
@@ -1076,6 +1176,7 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
}
|
||||
noteSends(ctx, p.Tier, []string{n}, carried, now)
|
||||
noteChanges(p, n, carried, was[n], now)
|
||||
}
|
||||
fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "),
|
||||
strings.Join(sent, ", "))
|
||||
@@ -1090,13 +1191,14 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return changed, nil
|
||||
}
|
||||
// And wait for what the next tier needs running.
|
||||
leftAsWas := ""
|
||||
needed := gates(*p, edges, rollsOut)
|
||||
if len(needed) > 0 {
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
var waiting []string
|
||||
var needs []gateNeed
|
||||
for _, m := range needed {
|
||||
running, err := inv.Running(ctx, m)
|
||||
if err != nil {
|
||||
@@ -1110,6 +1212,8 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
if state.State == planDeleted {
|
||||
continue
|
||||
}
|
||||
// A machine passed as away was sent nothing to wait on (ADR 0435 §1, issue 1161).
|
||||
running = runningHeard(*p, running, state)
|
||||
// The plan sends what it waits for. A rebuild from the same source commit is not a
|
||||
// move the catalogue announces — the build machine rebuilt for a controller change
|
||||
// is one — so the roll-out that opens this gate is the plan's to make, once, and
|
||||
@@ -1127,14 +1231,12 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
for _, r := range reports {
|
||||
byNode[r.Node] = r
|
||||
}
|
||||
if ok, on := appliedEach(m, func(node string) time.Time {
|
||||
needs = append(needs, gateNeed{m, running, func(node string) time.Time {
|
||||
if s := byNode[node].Sent; s != nil {
|
||||
return *s
|
||||
}
|
||||
return time.Time{}
|
||||
}, running, reports); !ok {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
||||
}
|
||||
}})
|
||||
continue
|
||||
}
|
||||
// **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218)
|
||||
@@ -1154,9 +1256,23 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
return at
|
||||
}
|
||||
if ok, on := appliedEach(m, sinceFor, running, reports); !ok {
|
||||
waiting = append(waiting, fmt.Sprintf("%s on %s", m, strings.Join(on, ", ")))
|
||||
}
|
||||
needs = append(needs, gateNeed{m, running, sinceFor})
|
||||
}
|
||||
waiting, quiet, lost := settleTierGate(needs, reports, silent)
|
||||
if lost {
|
||||
// Every machine the next tier waits on is silent: nothing it needs was delivered anywhere, so the plan does
|
||||
// not go on as if it had been (issue 1160).
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("tier %d built; every machine it waits on said nothing on its send within %s: %s",
|
||||
p.Tier, applySilentAfter, strings.Join(quiet, ", "))
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
return true, nil
|
||||
}
|
||||
if len(quiet) > 0 {
|
||||
fmt.Printf("%s: tier %d: %s said nothing on its send within %s and is left as it was; the plan goes on without it\n",
|
||||
p.ID, p.Tier, strings.Join(quiet, ", "), applySilentAfter)
|
||||
leftAsWas = fmt.Sprintf("%s said nothing on its send within %s and is left as it was", strings.Join(quiet, ", "),
|
||||
applySilentAfter)
|
||||
}
|
||||
if len(waiting) > 0 {
|
||||
note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(waiting, "; ") + " to be applied"
|
||||
@@ -1198,6 +1314,13 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
carryUserList(ctx, open, p)
|
||||
}
|
||||
}
|
||||
if end == "" {
|
||||
end = leftAsWas
|
||||
}
|
||||
// A machine passed as away is said where the others passed are said (ADR 0435 §1, issue 1161).
|
||||
if said := awaySaying(*p); said != "" {
|
||||
end = strings.TrimPrefix(end+"; "+said, "; ")
|
||||
}
|
||||
p.Tier++
|
||||
p.State = inventory.PlanBuilding
|
||||
p.Note = end
|
||||
@@ -1324,6 +1447,10 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
s := p.Modules[m]
|
||||
owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build})
|
||||
}
|
||||
// What the assignments name before the send, for the rollout to say what it changed; and the send is made as this
|
||||
// rollout, so what is recorded of it names it (novox/hq ADR 0408, issue 1095).
|
||||
was := ranBefore(ctx, inv, node)
|
||||
ctx = sendingAs(ctx, p)
|
||||
// A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges
|
||||
// all of it there.
|
||||
carried, sent, grants, err := gatedSend(ctx, open, node, owns)
|
||||
@@ -1350,6 +1477,9 @@ func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string
|
||||
}
|
||||
}
|
||||
noteSends(ctx, p.Tier, sent, modules, now)
|
||||
for _, n := range sent {
|
||||
noteChanges(p, n, modules, was, now)
|
||||
}
|
||||
p.State = inventory.PlanRolling
|
||||
p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "),
|
||||
strings.Join(sent, ", "))
|
||||
@@ -2016,6 +2146,10 @@ func plansCommand(ctx context.Context, args []string) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
// The assignments it changed, each with the build and commit before and after (novox/hq ADR 0408, issue 1095).
|
||||
for _, line := range changedLines(p) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if *whatIf != "" {
|
||||
|
||||
@@ -392,6 +392,32 @@ func rolloutMint(ctx context.Context, again bool) error {
|
||||
}
|
||||
modules++
|
||||
|
||||
case broker.KindProcess:
|
||||
// One process's own account (novox/hq ADR 0436 §1, §2), delivered as the module's own secret
|
||||
// broker-<process id> — minted here exactly as `module issue` mints it, so a process issued
|
||||
// once still gets a new password every time the bus's accounts are issued again.
|
||||
m, inShelf := shelf[p.Module]
|
||||
if !inShelf {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
pr, found := processPrincipalFor(m, p.Process)
|
||||
if !found {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
name := processSecretName(p.Process)
|
||||
if _, reads := m.OwnSecrets[name]; !reads {
|
||||
fmt.Printf(" %s's process %s declares bus but declares no `%s` secret to receive a "+
|
||||
"credential in; skipped\n", p.Module, p.Process, name)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if err := issueProcess(ctx, inv, m, pr, p.Node, "", known, busAddress); err != nil {
|
||||
return err
|
||||
}
|
||||
modules++
|
||||
|
||||
default:
|
||||
skipped++
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) {
|
||||
return s, heard
|
||||
}
|
||||
|
||||
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) {
|
||||
func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { // 1157/AC3: the send keys stay until release 6 (ADR 0433 §5)
|
||||
s, heard := silentOnABus(t)
|
||||
want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"}
|
||||
if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -408,14 +409,22 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
if err := need("members"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return []string{"delivery", "order", "--members", str("members")}, nil
|
||||
members, err := membersFlags(str("members"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]string{"delivery", "order"}, members...), nil
|
||||
case "delivery-check":
|
||||
if err := need("members"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := []string{"delivery", "check", "--members", str("members")}
|
||||
members, err := membersFlags(str("members"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
argv := append([]string{"delivery", "check"}, members...)
|
||||
if g := str("group"); g != "" {
|
||||
argv = []string{"delivery", "check", "--group", g, "--members", str("members")}
|
||||
argv = append([]string{"delivery", "check", "--group", g}, members...)
|
||||
}
|
||||
// Marked ready (novox/hq ADR 0406 §3): the check is queued before every other check.
|
||||
if r := strings.ToLower(str("ready")); r != "" && r != "false" && r != "no" && r != "0" {
|
||||
@@ -1193,6 +1202,7 @@ func runVerb(ctx context.Context, argv []string) (verbAnswer, error) {
|
||||
if handingOver.Load() {
|
||||
return verbAnswer{}, fmt.Errorf("%w: this controller is stopping and runs no new command", link.ErrHandingOver)
|
||||
}
|
||||
defer removeMembersFile(argv)
|
||||
cmd := selfCommand(ctx, argv)
|
||||
// And who asked, so an act it does by hand is recorded as theirs (novox/hq to-be 45 §7).
|
||||
caller := link.CallerIn(ctx)
|
||||
@@ -1858,3 +1868,41 @@ func terminalOnly(argv []string) error {
|
||||
"decide who may become root on it (novox/hq ADR 0266). A verb may run node list and node show. "+
|
||||
"Nothing was done", sub)
|
||||
}
|
||||
|
||||
// membersInArgument is how long a group's members may be as one argument of the command a verb re-executes: over
|
||||
// the kernel's 128 KiB for one argument the command cannot start (novox/hq issue 1272).
|
||||
const membersInArgument = 32 << 10
|
||||
|
||||
// membersDirPrefix names the private directory a long list of members is handed over in.
|
||||
const membersDirPrefix = "mesh-members-"
|
||||
|
||||
// membersFlags is a group's members as the flags of the command a verb re-executes: `--members <json>` while the
|
||||
// list is short, as a person would type it, and `--members-file <path>` to a file in a private directory of its own
|
||||
// when it is not. runVerb removes the directory once the command has ended.
|
||||
func membersFlags(members string) ([]string, error) {
|
||||
if len(members) <= membersInArgument {
|
||||
return []string{"--members", members}, nil
|
||||
}
|
||||
dir, err := os.MkdirTemp("", membersDirPrefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the members cannot be handed to the command: %w", err)
|
||||
}
|
||||
file := filepath.Join(dir, "members.json")
|
||||
if err := os.WriteFile(file, []byte(members), 0o600); err != nil {
|
||||
os.RemoveAll(dir)
|
||||
return nil, fmt.Errorf("the members cannot be handed to the command: %w", err)
|
||||
}
|
||||
return []string{"--members-file", file}, nil
|
||||
}
|
||||
|
||||
// removeMembersFile removes the private directory membersFlags made for a command line, and only such a directory.
|
||||
func removeMembersFile(argv []string) {
|
||||
for i, a := range argv {
|
||||
if a == "--members-file" && i+1 < len(argv) {
|
||||
dir := filepath.Dir(argv[i+1])
|
||||
if strings.HasPrefix(filepath.Base(dir), membersDirPrefix) {
|
||||
os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// A plan's send key the bus refuses (novox/hq ADR 0433 §4, issue 1157): the look that could not write it says which, and
|
||||
// a probe of the controller's self-check raises each as a condition naming the plan and the machine. The next look writes
|
||||
// the key again; the look that writes it leaves it out of the set, and the condition clears.
|
||||
const (
|
||||
kindSendKeyRefused = "send-key-refused"
|
||||
sendKeyRefusedID = "DL"
|
||||
)
|
||||
|
||||
var refusedSendKeys = struct {
|
||||
mu sync.Mutex
|
||||
by map[silentWait]error
|
||||
}{}
|
||||
|
||||
// noteRefusedSendKeys sets the keys the newest look could not write, replacing what the look before found.
|
||||
func noteRefusedSendKeys(refused map[silentWait]error) {
|
||||
refusedSendKeys.mu.Lock()
|
||||
defer refusedSendKeys.mu.Unlock()
|
||||
refusedSendKeys.by = refused
|
||||
}
|
||||
|
||||
// observeRefusedSendKeys is one condition per plan and machine whose send key was refused.
|
||||
func observeRefusedSendKeys() []conditions.Observation {
|
||||
refusedSendKeys.mu.Lock()
|
||||
waits := make([]silentWait, 0, len(refusedSendKeys.by))
|
||||
why := map[silentWait]error{}
|
||||
for w, err := range refusedSendKeys.by {
|
||||
waits = append(waits, w)
|
||||
why[w] = err
|
||||
}
|
||||
refusedSendKeys.mu.Unlock()
|
||||
sort.Slice(waits, func(i, j int) bool { return silentKey(waits[i]) < silentKey(waits[j]) })
|
||||
seen := map[[2]string]bool{}
|
||||
var out []conditions.Observation
|
||||
for _, w := range waits {
|
||||
if seen[[2]string{w.Plan, w.Node}] {
|
||||
continue // a plan sends a machine one tier at a time; one condition says it
|
||||
}
|
||||
seen[[2]string{w.Plan, w.Node}] = true
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.Plan, Token: kindSendKeyRefused + "-" + w.Node,
|
||||
Kind: kindSendKeyRefused, Machine: w.Node, Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("the bus refused the key that watches the delivery %s for a report from %s, so nothing "+
|
||||
"names the machine silent if it never reports; the plan writes the key again at its next look", w.Plan, w.Node),
|
||||
Said: fmt.Sprintf("%s: %v", silentKey(w), why[w])})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// probeSendKeyRefused reads what the newest look of the plans could not write.
|
||||
func probeSendKeyRefused(context.Context, *doctor) ([]conditions.Observation, error) {
|
||||
return observeRefusedSendKeys(), nil
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -38,6 +39,13 @@ type sendable struct {
|
||||
// the send; never on the wire.
|
||||
composedFrom int64
|
||||
actingEpoch uint64
|
||||
// ReadsModules says the machine's node-engine reads a module per resource and a build per module (novox/hq issue
|
||||
// 1145): only then are they sent, because an older node-engine refuses a key it does not know, whole.
|
||||
ReadsModules bool
|
||||
// moduleOf is the composer's owner of each resource id, and buildIDs the build each module carries (its id):
|
||||
// kept beside the resources, and on the wire only to a machine that reads them.
|
||||
moduleOf map[string]string
|
||||
buildIDs map[string]string
|
||||
// modules are the modules this declaration names, for the record of the send; never on the wire.
|
||||
modules []string
|
||||
// localNetwork is every port admitted from the local network and what it resolved to (novox/hq ADR
|
||||
@@ -111,6 +119,28 @@ type adoptionEnvelope struct {
|
||||
// Body is the declaration's bytes, as sent and as digested.
|
||||
func (s sendable) Body() ([]byte, error) {
|
||||
envelope := map[string]any{"declaration": 1, "resources": s.Resources}
|
||||
if s.ReadsModules {
|
||||
// The module of each resource the composer owns (what the mesh adds of its own has none), and the build of each
|
||||
// module (novox/hq issue 1145): to a machine that said it reads them, and no other.
|
||||
named := make([]map[string]any, len(s.Resources))
|
||||
for n, r := range s.Resources {
|
||||
module, owned := s.moduleOf[fmt.Sprint(r["id"])]
|
||||
if !owned {
|
||||
named[n] = r
|
||||
continue
|
||||
}
|
||||
with := make(map[string]any, len(r)+1)
|
||||
for k, v := range r {
|
||||
with[k] = v
|
||||
}
|
||||
with["module"] = module
|
||||
named[n] = with
|
||||
}
|
||||
envelope["resources"] = named
|
||||
if len(s.buildIDs) > 0 {
|
||||
envelope["builds"] = s.buildIDs
|
||||
}
|
||||
}
|
||||
if s.Adoption != nil {
|
||||
envelope["adoption"] = s.Adoption
|
||||
}
|
||||
|
||||
@@ -85,10 +85,7 @@ func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T)
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
mirror, err := inv.AssignmentsMirror(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mirror := inv.PlanValuesForTest(t)
|
||||
var value inventory.AssignmentMirrored
|
||||
if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil {
|
||||
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err)
|
||||
|
||||
@@ -318,6 +318,10 @@ func ago(d time.Duration) string {
|
||||
return d.Round(time.Minute).String()
|
||||
}
|
||||
|
||||
// silentBound is how long a machine may go unheard, where the watchman gives no verdict: three of the intervals nodes have
|
||||
// always used, fixed now that no machine says its own (issue 1183).
|
||||
const silentBound = heartbeatsMissed * heartbeatEvery
|
||||
|
||||
// heartbeatBound is a machine's S1 or S11 bound from the interval it says.
|
||||
func heartbeatBound(every time.Duration) time.Duration {
|
||||
if every <= 0 {
|
||||
@@ -333,9 +337,9 @@ func watchHeartbeats(f *signalFacts) []conditions.Observation {
|
||||
// Never heard is a machine that has not joined, which status says; asleep is not lost.
|
||||
continue
|
||||
}
|
||||
bound := heartbeatBound(m.every)
|
||||
bound := silentBound
|
||||
silent := f.now.Sub(m.lastHeard)
|
||||
if silent <= bound {
|
||||
if !m.silent(f.now) {
|
||||
continue
|
||||
}
|
||||
severity := conditions.Warning
|
||||
@@ -357,7 +361,7 @@ func watchReports(f *signalFacts) []conditions.Observation {
|
||||
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
|
||||
continue
|
||||
}
|
||||
if !m.lastHeard.IsZero() && f.now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
if m.silent(f.now) {
|
||||
continue // silent: S1 says it, and a silent machine reports nothing
|
||||
}
|
||||
bound := max(reportAtLeast, 3*m.lastApply)
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
func calm(now time.Time) *signalFacts {
|
||||
return &signalFacts{now: now, started: now.Add(-time.Hour), host: "anchor", toolsHeardFrom: now.Add(-time.Hour),
|
||||
machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second),
|
||||
every: time.Minute, sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||
sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute),
|
||||
lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}},
|
||||
plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2,
|
||||
entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}},
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1160/AC5, live, with the store and a real bus and no ticker: a plan of two tiers whose first tier has a machine that
|
||||
// never reports, and whose second tier waits on the first (the plan's own order), passes its first tier after the bound,
|
||||
// the silent key's expiry the only trigger, and sends its second tier to the machine that reported alone (issue 1160).
|
||||
func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToTheOthers(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
// A quarter of an hour of silence is two seconds here.
|
||||
wasSilent := applySilentAfter
|
||||
applySilentAfter = 2 * time.Second
|
||||
t.Cleanup(func() { applySilentAfter = wasSilent })
|
||||
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := js.EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacy, err := js.Conn().JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy}
|
||||
t.Cleanup(func() { checkEvents = nil })
|
||||
sentHeard := make(chan *nats.Msg, 32)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
|
||||
wasSaved := inventory.PlanSaved
|
||||
inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open)
|
||||
t.Cleanup(func() { inventory.PlanSaved = wasSaved })
|
||||
|
||||
// The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept.
|
||||
var mu sync.Mutex
|
||||
var asks []string
|
||||
var sends [][]string
|
||||
wasAsk := askABuild
|
||||
askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
id := link.NewBuildID(time.Now().Add(time.Duration(len(asks)) * time.Millisecond))
|
||||
asks = append(asks, source.Repository+"#"+id)
|
||||
return id, nil
|
||||
}
|
||||
t.Cleanup(func() { askABuild = wasAsk })
|
||||
digests := 0
|
||||
wasSend := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
sends = append(sends, append([]string(nil), names...))
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
builds := map[string]string{}
|
||||
for _, m := range []string{"a", "b"} {
|
||||
builds[m] = current[m].Commit
|
||||
}
|
||||
for _, node := range names {
|
||||
digests++
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = wasSend })
|
||||
|
||||
register := func(m, commit string, asked time.Time) {
|
||||
t.Helper()
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit},
|
||||
inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record := func(id, m, commit string, at time.Time) {
|
||||
t.Helper()
|
||||
b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at}
|
||||
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit})
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
for _, m := range []string{"a", "b"} {
|
||||
record("build-"+m+"-1", m, "c1", old)
|
||||
register(m, "c1", old)
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if _, err := inv.Assign(ctx, n, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, n := range []string{"anchor", "laptop"} {
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record("build-a-2", "a", "c2", time.Now().Add(-time.Minute))
|
||||
register("a", "c2", time.Now().Add(-time.Minute))
|
||||
|
||||
// A machine's report on what it was sent, heard as the controller hears it.
|
||||
listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}}
|
||||
seq := int64(0)
|
||||
reports := func(node string) {
|
||||
t.Helper()
|
||||
last, found, err := inv.SentTo(ctx, node)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s was sent nothing: %v", node, err)
|
||||
}
|
||||
seq++
|
||||
body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest,
|
||||
"report_sequence": seq})
|
||||
var report link.Report
|
||||
if err := json.Unmarshal(body, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := listener.Heard(ctx, report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
plan := func() inventory.Plan {
|
||||
t.Helper()
|
||||
p, err := inv.PlanByID(ctx, "plan-1160")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
built := time.Now().UTC()
|
||||
p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(),
|
||||
State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}},
|
||||
Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}},
|
||||
Modules: map[string]*inventory.PlanModule{
|
||||
"a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"},
|
||||
"b": {},
|
||||
}}
|
||||
first, stopFirst := context.WithCancel(ctx)
|
||||
defer stopFirst()
|
||||
startRolloutEvents(first, open, js)
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "tier 0 sent to both machines", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 1
|
||||
})
|
||||
reports("anchor")
|
||||
time.Sleep(time.Second)
|
||||
if got := plan().Tier; got != 0 {
|
||||
t.Fatalf("1160/AC5: the plan went on to tier %d before the bound, with laptop not heard from", got)
|
||||
}
|
||||
// The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own.
|
||||
var moved string
|
||||
eventually(t, "tier 1 passed once laptop's key expired", func() bool {
|
||||
if q := plan(); q.Tier == 1 {
|
||||
moved = q.Note
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
if !strings.Contains(moved, "laptop") || strings.Contains(moved, "anchor") {
|
||||
t.Errorf("1160/AC5: the plan says %q; want laptop named silent and not anchor", moved)
|
||||
}
|
||||
// The ask for tier 1's build is the next step, and records its build id on the module.
|
||||
eventually(t, "tier 1 asked", func() bool { m := plan().Modules["b"]; return m != nil && m.Build != "" })
|
||||
b := plan().Modules["b"]
|
||||
askedAt, _ := link.BuildAskedAt(b.Build)
|
||||
record(b.Build, "b", "c2", askedAt)
|
||||
register("b", "c2", askedAt)
|
||||
planBuilt(ctx, open, "b", "c2", "", askedAt, b.Build)
|
||||
eventually(t, "tier 1 sent to anchor alone", func() bool {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor"
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine silent past the bound at a tier between the first and the last is named silent and left as it was, as the
|
||||
// last tier's is (novox/hq ADR 0282 §1, ADR 0360 §4-5, issue 1160): one rule, walkReports, read by the walk's end and by
|
||||
// the gates between tiers.
|
||||
|
||||
func silentWalk(now time.Time) (inventory.Plan, []inventory.Reported) {
|
||||
sent := now.Add(-time.Minute)
|
||||
long := now.Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
at := now.Add(-10 * time.Second)
|
||||
p := inventory.Plan{Tier: 0, Tiers: [][]string{{"engine"}, {"app"}}, Modules: map[string]*inventory.PlanModule{
|
||||
"engine": {Rest: map[string]inventory.SentDeclaration{
|
||||
"anchor": {Digest: "d-anchor", At: sent},
|
||||
"g14": {Digest: "d-g14", At: long},
|
||||
"fresh": {Digest: "d-fresh", At: sent}}},
|
||||
"app": {}}}
|
||||
reports := []inventory.Reported{{Node: "anchor", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-anchor"}}
|
||||
return p, reports
|
||||
}
|
||||
|
||||
func needsOf(running ...string) []gateNeed {
|
||||
return []gateNeed{{module: "engine", running: running, since: func(string) time.Time { return time.Time{} }}}
|
||||
}
|
||||
|
||||
// 1160/AC1: at a middle tier the machine silent past the bound is named, not waited for, and the plan goes on with the
|
||||
// machines that reported; one still inside the bound is waited for.
|
||||
func TestAMiddleTierGateNamesTheSilentMachineAndGoesOnWithThoseThatReported(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
p, reports := silentWalk(now)
|
||||
silent := silentNodes(p, reports, now, nil)
|
||||
waiting, quiet, lost := settleTierGate(needsOf("anchor", "g14"), reports, silent)
|
||||
if len(waiting) != 0 || !slices.Equal(quiet, []string{"g14"}) || lost {
|
||||
t.Fatalf("1160/AC1: waiting %v, silent %v, lost %v; want g14 named silent and nothing waited for", waiting, quiet, lost)
|
||||
}
|
||||
waiting, quiet, _ = settleTierGate(needsOf("anchor", "g14", "fresh"), reports, silent)
|
||||
if len(waiting) != 1 || waiting[0] != "engine on fresh" || !slices.Equal(quiet, []string{"g14"}) {
|
||||
t.Fatalf("1160/AC1: a machine inside the bound is still waited for: waiting %v, silent %v", waiting, quiet)
|
||||
}
|
||||
}
|
||||
|
||||
// 1160/AC2: the walk's end and the gate read the same silent machines.
|
||||
func TestTheGateAndTheWalksEndShareOneRuleOfSilence(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
p, reports := silentWalk(now)
|
||||
_, _, ended := walkReports(p, reports, now, nil)
|
||||
got := silentNodes(p, reports, now, nil)
|
||||
if len(got) != len(ended) || !got["g14"] || !slices.Equal(ended, []string{"g14"}) {
|
||||
t.Fatalf("1160/AC2: the gate reads %v silent, the walk's end %v", got, ended)
|
||||
}
|
||||
}
|
||||
|
||||
// 1160/AC3: a later tier is not sent to a machine silent.
|
||||
func TestALaterTierIsNotSentToASilentMachine(t *testing.T) {
|
||||
got := withoutSilent([]string{"anchor", "g14"}, map[string]bool{"g14": true})
|
||||
if !slices.Equal(got, []string{"anchor"}) {
|
||||
t.Fatalf("1160/AC3: a later tier goes to %v; want anchor alone", got)
|
||||
}
|
||||
in := []string{"anchor", "g14"}
|
||||
withoutSilent(in, map[string]bool{"g14": true})
|
||||
if !slices.Equal(in, []string{"anchor", "g14"}) {
|
||||
t.Fatalf("1160/AC3: the machines running it were changed: %v", in)
|
||||
}
|
||||
}
|
||||
|
||||
// 1160/AC4: every machine the gate waits on silent: the plan is lost, not gone on as if delivered.
|
||||
func TestAGateWhoseEveryMachineIsSilentIsLost(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
p, reports := silentWalk(now)
|
||||
reports = nil
|
||||
p.Modules["engine"].Rest = map[string]inventory.SentDeclaration{
|
||||
"g14": {Digest: "d", At: now.Add(-time.Hour)}, "laptop": {Digest: "d", At: now.Add(-time.Hour)}}
|
||||
silent := silentNodes(p, reports, now, nil)
|
||||
_, quiet, lost := settleTierGate(needsOf("g14", "laptop"), reports, silent)
|
||||
if !lost || !slices.Equal(quiet, []string{"g14", "laptop"}) {
|
||||
t.Fatalf("1160/AC4: lost %v, silent %v; want both named and the gate lost", lost, quiet)
|
||||
}
|
||||
}
|
||||
@@ -414,18 +414,11 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
out.nodes, err = inv.Nodes(ctx)
|
||||
out.nodes, err = open.heardNodes(ctx)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
}
|
||||
for _, n := range out.nodes {
|
||||
// Never heard from, or not lately. Different from failing: a machine that says nothing
|
||||
// may be new, switched off, or unreachable, and none of those is a machine that tried
|
||||
// and could not.
|
||||
if n.LastSeen.IsZero() || time.Since(n.LastSeen) > time.Hour {
|
||||
out.quiet = append(out.quiet, n)
|
||||
}
|
||||
}
|
||||
out.quiet = notHeardLately(out.nodes, time.Now())
|
||||
out.behind, err = inv.Behind(ctx)
|
||||
if err != nil {
|
||||
return answers{}, err
|
||||
|
||||
@@ -132,6 +132,9 @@ func openInventory(ctx context.Context) (*inventory.Inventory, error) {
|
||||
}
|
||||
// And where config values are read once the move of config has run (issue 664): mesh-config_resolved.
|
||||
inv.ReadsConfigFrom(readConfigValues)
|
||||
// And the chain, opened when a kind that moved to the bus is first read (novox/hq ADR 0408): a command that reads none
|
||||
// has no use of a connection.
|
||||
inv.UseChainOpener(openTheChain)
|
||||
return inv, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
)
|
||||
|
||||
// What stopped moving (novox/hq ADR 0433 §2, §3, issue 1156). The watchman writes `stuck.<bucket>.<key>` in its own
|
||||
// bucket for each key whose pulse expired, and again as moved when the key moves; it never deletes one. The controller
|
||||
// reads that bucket in two places and holds no timer for either: a probe of the self-check raises each key not moved as
|
||||
// a condition of the delivery scope, and the check that a machine is silent takes a node's presence key as its verdict.
|
||||
const (
|
||||
kindStuck = "stuck"
|
||||
probeStuckID = "DM"
|
||||
)
|
||||
|
||||
// presenceVerdict is the watchman's word on whether a machine is online, from its stuck presence key.
|
||||
type presenceVerdict int
|
||||
|
||||
const (
|
||||
// presenceUnsaid: the watchman has written nothing for the machine; the heartbeat rule judges it, as before.
|
||||
presenceUnsaid presenceVerdict = iota
|
||||
// presenceHeard: the key says moved, the machine is back online.
|
||||
presenceHeard
|
||||
// presenceSilent: the key says not moved, the machine was offline past its bound.
|
||||
presenceSilent
|
||||
)
|
||||
|
||||
// presenceEntry is presence.<node> as the watchman wrote it: whether the machine is online and when the entry was made.
|
||||
// The watchman writes it only when it changes, so for an online machine the time is when it connected.
|
||||
type presenceEntry struct {
|
||||
Online bool
|
||||
At time.Time
|
||||
}
|
||||
|
||||
// applyPresence sets what the watchman says of a machine: the verdict of its stuck presence key (nil when there is none).
|
||||
// Last heard is lastHeard's: now when online, the entry's time when offline, the store's last report only without an entry.
|
||||
func applyPresence(m *machineFacts, stuck *sdkdelivery.Stuck, entry *presenceEntry, now time.Time) {
|
||||
switch {
|
||||
case stuck == nil:
|
||||
m.presence = presenceUnsaid
|
||||
case stuck.Moved:
|
||||
m.presence = presenceHeard
|
||||
default:
|
||||
m.presence = presenceSilent
|
||||
}
|
||||
m.lastHeard = lastHeard(m.lastHeard, entry, now)
|
||||
}
|
||||
|
||||
// silent says the machine is silent: the watchman's verdict where it gave one, else the heartbeat rule. A machine never
|
||||
// heard from is not silent, it has not joined.
|
||||
func (m machineFacts) silent(now time.Time) bool {
|
||||
switch m.presence {
|
||||
case presenceSilent:
|
||||
return true
|
||||
case presenceHeard:
|
||||
return false
|
||||
}
|
||||
return !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > silentBound
|
||||
}
|
||||
|
||||
// stuckObservations are the conditions of what stopped moving: one for each key not moved, in the delivery scope, kind
|
||||
// stuck, naming the key and what it waited for. A node's presence key is left to the silent check; a moved key raises
|
||||
// nothing, so its condition clears.
|
||||
func stuckObservations(entries map[string]sdkdelivery.Stuck) []conditions.Observation {
|
||||
keys := make([]string, 0, len(entries))
|
||||
for k := range entries {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
var out []conditions.Observation
|
||||
for _, k := range keys {
|
||||
s := entries[k]
|
||||
if s.Moved {
|
||||
continue
|
||||
}
|
||||
if s.Bucket == sdkdelivery.PresenceBucket {
|
||||
if node, _, _, err := sdkdelivery.ParsePresenceKey(s.Key); err == nil && node != "" {
|
||||
continue
|
||||
}
|
||||
}
|
||||
waited := s.Waited
|
||||
if waited == "" {
|
||||
waited = "a move"
|
||||
}
|
||||
out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: k, Kind: kindStuck,
|
||||
Severity: conditions.Warning,
|
||||
Summary: fmt.Sprintf("%s has not moved since %s: it waited for %s", k, s.Since.UTC().Format("2006-01-02 15:04 MST"),
|
||||
waited),
|
||||
Said: fmt.Sprintf("%s: waiting for %s since %s", k, waited, s.Since.UTC().Format(time.RFC3339))})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// readStuck is every key of the stuck bucket and its value; none when the bucket is not there yet (the watchman not
|
||||
// delivered). A value of another version is left out, said.
|
||||
func readStuck(ctx context.Context, conn *nats.Conn) (map[string]sdkdelivery.Stuck, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watchman's stuck bucket: %w", err)
|
||||
}
|
||||
keys, err := boundedCall(ctx, "listing the stuck keys", func(ctx context.Context) ([]string, error) { return kv.Keys(ctx) })
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]sdkdelivery.Stuck{}
|
||||
for _, key := range keys {
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s, err := sdkdelivery.DecodeStuck(e.Value())
|
||||
if err != nil {
|
||||
fmt.Printf("the stuck key %s is not read: %v\n", key, err)
|
||||
continue
|
||||
}
|
||||
out[key] = s
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// readPresence is, by node, presence.<node>'s entry; a node without an entry is left out.
|
||||
func readPresence(ctx context.Context, conn *nats.Conn, nodes []string) (map[string]presenceEntry, error) {
|
||||
api, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.PresenceBucket)
|
||||
if errors.Is(err, jetstream.ErrBucketNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the watchman's presence bucket: %w", err)
|
||||
}
|
||||
out := map[string]presenceEntry{}
|
||||
for _, node := range nodes {
|
||||
key, err := sdkdelivery.PresenceNodeKey(node)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
e, err := kv.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p, err := sdkdelivery.DecodePresence(e.Value())
|
||||
if err != nil {
|
||||
fmt.Printf("the presence key %s is not read: %v\n", key, err)
|
||||
continue
|
||||
}
|
||||
out[node] = presenceEntry{Online: p.Online, At: e.Created()}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// withPresence says of each machine what the watchman says: its stuck presence key and its presence entry's time.
|
||||
func withPresence(ctx context.Context, conn *nats.Conn, machines []machineFacts, now time.Time) error {
|
||||
stuck, err := readStuck(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, len(machines))
|
||||
for i, m := range machines {
|
||||
names[i] = m.name
|
||||
}
|
||||
entries, err := readPresence(ctx, conn, names)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range machines {
|
||||
var said *sdkdelivery.Stuck
|
||||
if key, err := sdkdelivery.StuckKey(sdkdelivery.PresenceBucket, machines[i].name); err == nil {
|
||||
if s, ok := stuck[key]; ok {
|
||||
said = &s
|
||||
}
|
||||
}
|
||||
var entry *presenceEntry
|
||||
if e, ok := entries[machines[i].name]; ok {
|
||||
entry = &e
|
||||
}
|
||||
applyPresence(&machines[i], said, entry, now)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// probeStuck is DM: every key the watchman says stopped moving is raised, but a node's presence, which the silent check
|
||||
// takes.
|
||||
func probeStuck(ctx context.Context, d *doctor) ([]conditions.Observation, error) {
|
||||
if d.js == nil {
|
||||
return nil, nil
|
||||
}
|
||||
entries, err := readStuck(ctx, d.js.Conn())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stuckObservations(entries), nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1156/AC4: on the embedded bus with the watchman's buckets, a stuck key written raises a condition and written as moved
|
||||
// clears it; a node's presence key not moved says the machine silent through the facts, and moved says it heard.
|
||||
func TestAStuckKeyWrittenAndThenMovedRaisesAndClearsACondition(t *testing.T) { // 1156/AC4
|
||||
js, err := broker.Dial(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if _, err := broker.RaiseBuckets(js, []broker.Bucket{
|
||||
{Module: sdkdelivery.WatchmanModule, Name: "stuck"},
|
||||
{Module: sdkdelivery.WatchmanModule, Name: "presence", History: sdkdelivery.PresenceHistory},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
api, err := jetstream.New(js.Conn())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, sdkdelivery.StuckBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
write := func(bucket, key string, moved bool) string {
|
||||
t.Helper()
|
||||
stuckKey, err := sdkdelivery.StuckKey(bucket, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := sdkdelivery.EncodeStuck(sdkdelivery.Stuck{Bucket: bucket, Key: key, Waited: "an answer", Since: time.Now().UTC(), Moved: moved})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := kv.Put(ctx, stuckKey, body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return stuckKey
|
||||
}
|
||||
d := &doctor{js: js}
|
||||
stuckKey := write(sdkdelivery.PulseBucket, "builds.x", false)
|
||||
write(sdkdelivery.PresenceBucket, "ace", false)
|
||||
got, err := probeStuck(ctx, d)
|
||||
if err != nil || len(got) != 1 || got[0].ID != stuckKey || got[0].Kind != kindStuck {
|
||||
t.Fatalf("a stuck key written: %+v %v", got, err)
|
||||
}
|
||||
machines := []machineFacts{{name: "ace", lastHeard: time.Now()}}
|
||||
if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !machines[0].silent(time.Now()) {
|
||||
t.Fatal("a presence key not moved did not say the machine silent")
|
||||
}
|
||||
write(sdkdelivery.PulseBucket, "builds.x", true)
|
||||
write(sdkdelivery.PresenceBucket, "ace", true)
|
||||
if got, err := probeStuck(ctx, d); err != nil || len(got) != 0 {
|
||||
t.Fatalf("a stuck key written as moved: %+v %v", got, err)
|
||||
}
|
||||
machines = []machineFacts{{name: "ace", lastHeard: time.Now().Add(-time.Hour)}}
|
||||
if err := withPresence(ctx, js.Conn(), machines, time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if machines[0].silent(time.Now()) {
|
||||
t.Fatal("a presence key that moved still says the machine silent")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func aStuckKey(t *testing.T, bucket, key string) string {
|
||||
t.Helper()
|
||||
k, err := sdkdelivery.StuckKey(bucket, key)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// 1156/AC1: each not-moved stuck key that is not a node's presence is an observation of the delivery scope, kind stuck,
|
||||
// naming the key and what it waited for; a moved key raises nothing, so its condition clears.
|
||||
func TestEachStuckKeyThatHasNotMovedIsAConditionOfTheDeliveryScope(t *testing.T) { // 1156/AC1
|
||||
since := time.Date(2026, 10, 18, 19, 0, 0, 0, time.UTC)
|
||||
pulse := aStuckKey(t, sdkdelivery.PulseBucket, "x")
|
||||
moved := aStuckKey(t, sdkdelivery.HaveBucket, "ace.zsh")
|
||||
node := aStuckKey(t, sdkdelivery.PresenceBucket, "ace")
|
||||
part := aStuckKey(t, sdkdelivery.PresenceBucket, "part.mesh-controller.one")
|
||||
entries := map[string]sdkdelivery.Stuck{
|
||||
pulse: {Bucket: sdkdelivery.PulseBucket, Key: "x", Waited: "the build of zsh to be answered", Since: since},
|
||||
moved: {Bucket: sdkdelivery.HaveBucket, Key: "ace.zsh", Waited: "a want", Since: since, Moved: true},
|
||||
node: {Bucket: sdkdelivery.PresenceBucket, Key: "ace", Waited: "its return", Since: since},
|
||||
part: {Bucket: sdkdelivery.PresenceBucket, Key: "part.mesh-controller.one", Waited: "its return", Since: since},
|
||||
}
|
||||
got := stuckObservations(entries)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("observations %+v, want the pulse key and the part's presence", got)
|
||||
}
|
||||
byID := map[string]conditions.Observation{}
|
||||
for _, o := range got {
|
||||
if o.Scope != conditions.ScopeDelivery || o.Kind != kindStuck {
|
||||
t.Errorf("%+v is not a stuck condition of the delivery scope", o)
|
||||
}
|
||||
byID[o.ID] = o
|
||||
}
|
||||
o, ok := byID[pulse]
|
||||
if !ok || !strings.Contains(o.Summary, pulse) || !strings.Contains(o.Summary, "the build of zsh to be answered") {
|
||||
t.Fatalf("the condition does not name the key and what it waited for: %+v", got)
|
||||
}
|
||||
if _, ok := byID[part]; !ok {
|
||||
t.Errorf("a part's presence is left out: %v", byID)
|
||||
}
|
||||
// Moved again: the same key, written as moved, raises nothing.
|
||||
entries[pulse] = sdkdelivery.Stuck{Bucket: sdkdelivery.PulseBucket, Key: "x", Since: since, Moved: true}
|
||||
if got := stuckObservations(entries); len(got) != 1 {
|
||||
t.Fatalf("a moved key still raises: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func aMachineHeardAt(now time.Time, ago time.Duration) machineFacts {
|
||||
return machineFacts{name: "laptop", lastHeard: now.Add(-ago)}
|
||||
}
|
||||
|
||||
// 1156/AC2: a node's stuck presence key not moved makes the machine silent, unless it sleeps; moved, it is heard; no
|
||||
// timer is armed (the verdict is the key's, whatever the time of the last word).
|
||||
func TestAStuckPresenceKeyIsTheSilentVerdict(t *testing.T) { // 1156/AC2
|
||||
now := time.Now()
|
||||
f := calm(now)
|
||||
f.machines = append(f.machines, aMachineHeardAt(now, 10*time.Second))
|
||||
notMoved := sdkdelivery.Stuck{Bucket: sdkdelivery.PresenceBucket, Key: "laptop", Waited: "its return", Since: now.Add(-3 * time.Minute)}
|
||||
applyPresence(&f.machines[1], ¬Moved, nil, now)
|
||||
got := watchHeartbeats(f)
|
||||
if len(got) != 1 || got[0].Machine != "laptop" || got[0].Kind != "silent" {
|
||||
t.Fatalf("a presence key not moved: %+v", got)
|
||||
}
|
||||
// Asleep: not silent.
|
||||
f.machines[1].power = link.PowerState{State: "sleeping", At: now.Add(-time.Hour)}
|
||||
if got := watchHeartbeats(f); len(got) != 0 {
|
||||
t.Fatalf("a sleeping machine raised %+v", got)
|
||||
}
|
||||
// Moved: heard, whatever the store's last word says.
|
||||
f.machines[1].power = link.PowerState{}
|
||||
f.machines[1].lastHeard = now.Add(-2 * time.Hour)
|
||||
moved := notMoved
|
||||
moved.Moved = true
|
||||
applyPresence(&f.machines[1], &moved, nil, now)
|
||||
if got := watchHeartbeats(f); len(got) != 0 {
|
||||
t.Fatalf("a presence key that moved raised %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1156/AC3: with a presence entry that says offline, last heard is the entry's time (1183/AC1); with none,
|
||||
// last_seen. An entry that says online counts as heard now: the watchman writes it only when it changes, so its time is
|
||||
// when the machine connected, and a machine online for hours with no stuck key is not silent, whatever last_seen says.
|
||||
func TestLastHeardFollowsThePresenceEntry(t *testing.T) { // 1156/AC3
|
||||
now := time.Now()
|
||||
seen := now.Add(-time.Hour)
|
||||
m := machineFacts{name: "laptop", lastHeard: seen}
|
||||
applyPresence(&m, nil, nil, now)
|
||||
if !m.lastHeard.Equal(seen) {
|
||||
t.Fatalf("with no presence entry last heard is %v, want last_seen %v", m.lastHeard, seen)
|
||||
}
|
||||
offlineAt := now.Add(-5 * time.Minute)
|
||||
applyPresence(&m, nil, &presenceEntry{Online: false, At: offlineAt}, now)
|
||||
if !m.lastHeard.Equal(offlineAt) {
|
||||
t.Fatalf("offline since %v: last heard is %v", offlineAt, m.lastHeard)
|
||||
}
|
||||
m = machineFacts{name: "laptop", lastHeard: now.Add(-time.Minute)}
|
||||
applyPresence(&m, nil, &presenceEntry{Online: false, At: now.Add(-time.Hour)}, now)
|
||||
if !m.lastHeard.Equal(now.Add(-time.Hour)) {
|
||||
t.Fatalf("an offline entry's time is last heard, not a report: %v", m.lastHeard)
|
||||
}
|
||||
for _, lastSeen := range []time.Duration{10 * time.Second, 5 * time.Hour} {
|
||||
m := machineFacts{name: "laptop", lastHeard: now.Add(-lastSeen)}
|
||||
applyPresence(&m, nil, &presenceEntry{Online: true, At: now.Add(-6 * time.Hour)}, now)
|
||||
if m.silent(now) {
|
||||
t.Fatalf("a machine online since hours ago (last_seen %v ago) with no stuck key is silent", lastSeen)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -303,6 +303,16 @@ func notNow(err error) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// unsaved marks the failure to save the plan a merge was added to, whatever it was: the merge is carried until that save
|
||||
// writes it (inventory.AddMerge), so the announcement is held and handed over again, never taken (novox/hq ADR 0407
|
||||
// decision 3, ADR 0408, issue 1092). The bus acknowledges it only after the save succeeded.
|
||||
func unsaved(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%w: the merge is not kept until its plan is saved: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
|
||||
// SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which
|
||||
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
@@ -396,7 +406,12 @@ func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceM
|
||||
if len(added) == 0 || len(from) == 0 {
|
||||
return nil
|
||||
}
|
||||
source := buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}
|
||||
return askBuilds(ctx, inv, m, buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat}, added)
|
||||
}
|
||||
|
||||
// askBuilds asks the first build of each directory of a merge's repository, as askNewModules does, from the source given.
|
||||
func askBuilds(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, source buildSource,
|
||||
added []string) []string {
|
||||
var asked []string
|
||||
for _, dir := range added {
|
||||
path := dir
|
||||
@@ -409,7 +424,7 @@ func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceM
|
||||
request := inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path,
|
||||
Ref: m.Base, Commit: m.Commit, For: "merge"}
|
||||
if err != nil {
|
||||
request.ID = fmt.Sprintf("not-asked-%s-%s", short(m.Commit), strings.ReplaceAll(dir, "/", "-"))
|
||||
request.ID = notAskedID(m.Commit, dir)
|
||||
request.NotAsked = err.Error()
|
||||
recordBuildRequest(ctx, inv, request)
|
||||
fmt.Printf(" %s is a new module in %s/%s and could not be built: %v — a hand `build` of it "+
|
||||
@@ -424,6 +439,15 @@ func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceM
|
||||
return asked
|
||||
}
|
||||
|
||||
// notAskedID is the id a merge's build that could not be asked is kept under: a key the bus takes, so a module at the
|
||||
// repository's root (".") is kept as one in a folder is.
|
||||
func notAskedID(commit, dir string) string {
|
||||
if dir == "." || dir == "" {
|
||||
return "not-asked-" + short(commit)
|
||||
}
|
||||
return fmt.Sprintf("not-asked-%s-%s", short(commit), strings.ReplaceAll(strings.Trim(dir, "/"), "/", "-"))
|
||||
}
|
||||
|
||||
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
|
||||
var actingOnMerges sync.Mutex
|
||||
|
||||
|
||||
@@ -142,9 +142,7 @@ type machineFacts struct {
|
||||
control bool
|
||||
// lastHeard is the store's last word from it, any word; zero when never.
|
||||
lastHeard time.Time
|
||||
// every is the heartbeat interval it said; zero when it said none.
|
||||
every time.Duration
|
||||
power link.PowerState
|
||||
power link.PowerState
|
||||
// sentAt is when it was last sent a declaration; reportedCurrent whether it has reported that one.
|
||||
sentAt time.Time
|
||||
reportedCurrent bool
|
||||
@@ -155,6 +153,8 @@ type machineFacts struct {
|
||||
tools bool
|
||||
toolsHeard time.Time
|
||||
toolsEvery time.Duration
|
||||
// presence is the watchman's word on whether it is online (novox/hq ADR 0433 §3); unsaid, the heartbeat rule judges.
|
||||
presence presenceVerdict
|
||||
}
|
||||
|
||||
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||
@@ -455,9 +455,6 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
for _, n := range nodes {
|
||||
m := machineFacts{name: n.Name, control: slices.Contains(control, n.Name), lastHeard: n.LastSeen,
|
||||
lastApply: lastApply[n.Name], tools: slices.Contains(tooled, n.Name)}
|
||||
if beat, ok := link.HostBeats.Of(n.Name); ok {
|
||||
m.every = beat.Every
|
||||
}
|
||||
if beat, ok := link.ToolsBeats.Of(n.Name); ok {
|
||||
m.toolsHeard, m.toolsEvery = beat.At, beat.Every
|
||||
}
|
||||
@@ -470,7 +467,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
m.reportedCurrent = r.Current
|
||||
}
|
||||
if !m.lastHeard.IsZero() && now.Sub(m.lastHeard) > heartbeatBound(m.every) {
|
||||
if m.silent(now) {
|
||||
anyPast = true
|
||||
}
|
||||
if m.tools && now.Sub(later(m.toolsHeard, link.ToolsBeats.Started())) > heartbeatBound(m.toolsEvery) {
|
||||
@@ -478,6 +475,19 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
// What the watchman says of each machine: its stuck presence key is the silent verdict, and presence.<node>'s time when
|
||||
// it was last heard (ADR 0433 §3). Unreadable is said, and the heartbeat rule judges as before.
|
||||
if w.js != nil {
|
||||
if err := withPresence(ctx, w.js.Conn(), out, now); err != nil {
|
||||
fmt.Printf("what the watchman says of the machines' presence cannot be read: %v\n", err)
|
||||
} else {
|
||||
for _, m := range out {
|
||||
if m.silent(now) {
|
||||
anyPast = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// What a machine past its bound last said of its power, read only then: a machine that said it
|
||||
// is asleep is not lost (ADR 0211). Unreadable is said: a sleeping laptop is then called silent,
|
||||
// which is the louder mistake and the right one to make.
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-controller
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
@@ -36,4 +36,4 @@ require (
|
||||
// committed. Every build (the build agent's `go build`, the Dockerfile) compiles from vendor/ and
|
||||
// fetches nothing; go refuses to build when vendor/ and this file disagree, so a pin moved without
|
||||
// `go mod vendor` fails loudly, at once, everywhere.
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261017162006-6469e8238443
|
||||
replace github.com/novox/mesh-host => git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261017162006-6469e8238443 h1:zUtpGbiH9DLjCPY+KJsm+PZ25yW6k/NmGVS3haGFfMM=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261017162006-6469e8238443/go.mod h1:5QOm3cIfu/k9fN6QqgDbKGCcDst7Ul1UlcfsiMVm9j4=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4=
|
||||
git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4 h1:yYDxFRJoH59kiCoEulD+FBwctyf9CrtKIxlEAsVL39Q=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261018184557-238440d6f8c4/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9 h1:qHpN/fpTDqijfMVoepxUT6mPkfcydb4LiPKsIokv6iM=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019091011-aa68b67389c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
|
||||
@@ -97,27 +97,17 @@ func theCataloguesEvents(t *testing.T) ([]AnEmitter, []AConsumer, []DeclaredSeat
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Seats []struct {
|
||||
Name string `json:"name"`
|
||||
Emits []string `json:"emits"`
|
||||
} `json:"seats"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
emitter, consumer, declared, err := eventsOfManifest(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", e.Name(), err)
|
||||
}
|
||||
if len(m.Emits) > 0 {
|
||||
emitters = append(emitters, AnEmitter{Module: m.Module, Emits: m.Emits})
|
||||
if emitter != nil {
|
||||
emitters = append(emitters, *emitter)
|
||||
}
|
||||
if len(m.Consumes) > 0 {
|
||||
consumers = append(consumers, AConsumer{Module: m.Module, Consumes: m.Consumes})
|
||||
}
|
||||
for _, s := range m.Seats {
|
||||
seats = append(seats, DeclaredSeat{Name: s.Name, Emits: s.Emits})
|
||||
if consumer != nil {
|
||||
consumers = append(consumers, *consumer)
|
||||
}
|
||||
seats = append(seats, declared...)
|
||||
}
|
||||
if len(emitters) == 0 {
|
||||
t.Fatalf("no module under %s emits anything, so this proved nothing", root)
|
||||
@@ -245,3 +235,41 @@ func TestTheBrokerAndTheCatalogueAgreeOnTheRuntimeModule(t *testing.T) {
|
||||
t.Fatalf("the broker calls the runtime %q and the catalogue %q", RuntimeModule, catalogue.RuntimeModule)
|
||||
}
|
||||
}
|
||||
|
||||
// eventsOfManifest is what one manifest says it emits and consumes for the agreement: its `emits`, and the events of
|
||||
// its own streams, which a consumer names as <module>.<event> as it names the others (novox/hq issue 1249: once
|
||||
// mesh-mail declared emits beside its stream, the stream's events were no longer counted).
|
||||
func eventsOfManifest(raw []byte) (*AnEmitter, *AConsumer, []DeclaredSeat, error) {
|
||||
var m struct {
|
||||
Module string `json:"module"`
|
||||
Emits []string `json:"emits"`
|
||||
Consumes []string `json:"consumes"`
|
||||
Streams []struct {
|
||||
Events []string `json:"events"`
|
||||
} `json:"streams"`
|
||||
Seats []struct {
|
||||
Name string `json:"name"`
|
||||
Emits []string `json:"emits"`
|
||||
} `json:"seats"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
emits := append([]string{}, m.Emits...)
|
||||
for _, s := range m.Streams {
|
||||
emits = append(emits, s.Events...)
|
||||
}
|
||||
var emitter *AnEmitter
|
||||
if len(emits) > 0 {
|
||||
emitter = &AnEmitter{Module: m.Module, Emits: emits}
|
||||
}
|
||||
var consumer *AConsumer
|
||||
if len(m.Consumes) > 0 {
|
||||
consumer = &AConsumer{Module: m.Module, Consumes: m.Consumes}
|
||||
}
|
||||
var seats []DeclaredSeat
|
||||
for _, s := range m.Seats {
|
||||
seats = append(seats, DeclaredSeat{Name: s.Name, Emits: s.Emits})
|
||||
}
|
||||
return emitter, consumer, seats, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mailShaped is mesh-mail's manifest as mesh-catalog main (b767aac8) declares its events: emits beside two streams of
|
||||
// its own (modules/mesh-mail/module.json), trimmed to the keys the agreement reads.
|
||||
const mailShaped = `{"module": "mesh-mail",
|
||||
"emits": ["mail.send.sent", "mail.send.failed"],
|
||||
"streams": [
|
||||
{"name": "mail", "events": ["mail.*.message.added", "mail.*.message.changed", "mail.*.message.removed"]},
|
||||
{"name": "mail-sends", "events": ["mail.send.approved"]}]}`
|
||||
|
||||
func TestAModulesStreamEventsCountBesideItsEmits(t *testing.T) { // 1249/AC1
|
||||
emitter, _, _, err := eventsOfManifest([]byte(mailShaped))
|
||||
if err != nil || emitter == nil {
|
||||
t.Fatalf("emitter %v, err %v", emitter, err)
|
||||
}
|
||||
bad := Disagreements([]AnEmitter{*emitter},
|
||||
[]AConsumer{{Module: "mesh-mail-notifier", Consumes: []string{"mesh-mail.mail.*.message.added"}}}, nil)
|
||||
if len(bad) != 0 {
|
||||
t.Fatalf("a consumer of mesh-mail's stream event was reported: %v", bad)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEventNeitherEmittedNorOnTheStreamIsStillCaught(t *testing.T) { // 1249/AC2
|
||||
emitter, _, _, err := eventsOfManifest([]byte(mailShaped))
|
||||
if err != nil || emitter == nil {
|
||||
t.Fatalf("emitter %v, err %v", emitter, err)
|
||||
}
|
||||
bad := Disagreements([]AnEmitter{*emitter},
|
||||
[]AConsumer{{Module: "mesh-mail-notifier", Consumes: []string{"mesh-mail.mail.*.message.renamed"}}}, nil)
|
||||
if len(bad) != 1 || !strings.Contains(bad[0], "mail.*.message.added") {
|
||||
t.Fatalf("want one disagreement naming the stream's events among what mesh-mail emits, got %v", bad)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 1155/AC1: a module whose manifest reads mesh-controller.checks-running may watch and get every key of the bucket and
|
||||
// write none.
|
||||
func TestAModuleReadingChecksRunningReadsAndWritesNone(t *testing.T) { // 1155/AC1
|
||||
if !slices.Contains(ControllerMirrors(), "checks-running") || BucketName(ControllerSeat, "checks-running") != CheckRunningBucket {
|
||||
t.Fatalf("a module cannot name %s in `reads`: %v", CheckRunningBucket, ControllerMirrors())
|
||||
}
|
||||
perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-watchman", PasswordHash: "x",
|
||||
Reads: []string{"mesh-controller.checks-running"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stream := "KV_" + CheckRunningBucket
|
||||
has(t, perms.Publish, "$JS.API.STREAM.INFO."+stream)
|
||||
has(t, perms.Publish, "$JS.API.DIRECT.GET."+stream+".>")
|
||||
has(t, perms.Publish, "$JS.API.CONSUMER.CREATE."+stream+".>")
|
||||
for _, p := range perms.Publish {
|
||||
if SubjectsOverlap(p, "$KV."+CheckRunningBucket+".any.key") {
|
||||
t.Errorf("the reading module may write %s through %q", CheckRunningBucket, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1155/AC3: the writers table names checks-running's writer as the build seat's holders, and the rule that a mirror is
|
||||
// written by the controller alone names it as the exception, with its reason.
|
||||
func TestChecksRunningIsAMirrorWrittenByTheBuildSeatsHolders(t *testing.T) { // 1155/AC3
|
||||
why := MirrorsWrittenByAnother["checks-running"]
|
||||
if why == "" {
|
||||
t.Fatal("the mirrors written by another do not name checks-running with a reason")
|
||||
}
|
||||
subject := "$KV." + CheckRunningBucket + ".novox.a.7"
|
||||
var rows int
|
||||
for _, row := range WritersTable {
|
||||
for _, s := range row.Subjects {
|
||||
if s != "$KV."+CheckRunningBucket+".>" {
|
||||
continue
|
||||
}
|
||||
rows++
|
||||
holder := Principal{Kind: KindModule, Module: "builder", Node: "one", Holds: []Seat{{Name: TheBuildSeat}}}
|
||||
other := Principal{Kind: KindModule, Module: "mesh-delivery-watchman", Node: "one"}
|
||||
if row.Writes == nil || !row.Writes(holder, subject) || row.Writes(other, subject) {
|
||||
t.Errorf("%q is not written by the build seat's holders alone", row.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("%d rows of the writers table write %s, want one", rows, CheckRunningBucket)
|
||||
}
|
||||
for m := range MirrorsWrittenByAnother {
|
||||
if m != "checks-running" {
|
||||
t.Errorf("the mirror %q is written by another and is not checks-running", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user