jschoubben
71be15f530
With the toolchain agreed by majority, a node on the minority version can pull an ask for the agreed one. It now declines it (the issue 890 path) for the asker to put again, so it never fails a build. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL
264 lines
11 KiB
Go
264 lines
11 KiB
Go
package buildrun
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
)
|
|
|
|
func gateReport(t *testing.T, node, version string, at time.Time) (string, []byte) {
|
|
t.Helper()
|
|
key, err := delivery.ToolchainKey(node, "typescript")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: at})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return key, value
|
|
}
|
|
|
|
// placing is a value of the state `placement` placing modules on node.
|
|
func placing(t *testing.T, node string, modules ...string) (string, []byte) {
|
|
t.Helper()
|
|
p := delivery.Placement{Node: node}
|
|
for _, m := range modules {
|
|
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
|
}
|
|
values, err := delivery.PlacementValues(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
key, _ := delivery.PlacementNodeKey(node)
|
|
return key, values[key]
|
|
}
|
|
|
|
// placingLegacy is placing in the form before, `placement.<node>` (research 072 row 37).
|
|
func placingLegacy(t *testing.T, node string, modules ...string) (string, []byte) {
|
|
t.Helper()
|
|
p := delivery.Placement{Node: node}
|
|
for _, m := range modules {
|
|
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
|
}
|
|
key, _ := delivery.LegacyPlacementKey(node)
|
|
value, err := delivery.EncodePlacement(p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return key, value
|
|
}
|
|
|
|
func agreedOf(set *ToolchainSet) (string, string) {
|
|
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
|
return tc.Version, none
|
|
}
|
|
|
|
// Issue 1940, the gate over the catalogue's placement (issue 774, ADR 0549 §3): a build node's report counts only while
|
|
// the placement places its build module on its node, and only once written since; before the placement is settled, and
|
|
// while no node's placement is held, every report counts. The live cases: mesh-build-node taken off ace left
|
|
// ace.typescript at v26.12.0 against v26.12.1 on the three others; put on ace again, its report of 2026-10-23 06:29 came
|
|
// back before the module ran there.
|
|
func TestToolchainGate(t *testing.T) {
|
|
old := time.Date(2026, 10, 23, 6, 29, 18, 0, time.UTC) // ace's report from its earlier run
|
|
placed := time.Date(2026, 10, 25, 19, 56, 45, 0, time.UTC) // mesh-build-node placed on ace again
|
|
three := []string{"g14", "novox", "shanks"}
|
|
start := func(t *testing.T, seeded bool) (*ToolchainGate, *ToolchainSet, *[]string) {
|
|
set := &ToolchainSet{}
|
|
said := &[]string{}
|
|
gate := NewToolchainGate([]string{GoBuildModule, NodeBuildModule}, set.Put, func(f string, a ...any) { *said = append(*said, fmt.Sprintf(f, a...)) })
|
|
for _, node := range three {
|
|
if seeded {
|
|
key, value := placing(t, node, NodeBuildModule)
|
|
gate.Placement(key, value, false, old.Add(-time.Hour))
|
|
}
|
|
key, value := gateReport(t, node, "v26.12.1", placed.Add(time.Minute))
|
|
gate.Toolchain(NodeBuildModule, key, value, false)
|
|
}
|
|
return gate, set, said
|
|
}
|
|
|
|
t.Run("not placed on ace: its report is no builder's", func(t *testing.T) {
|
|
gate, set, said := start(t, true)
|
|
key, value := gateReport(t, "ace", "v26.12.0", old)
|
|
gate.Toolchain(NodeBuildModule, key, value, false)
|
|
if a := delivery.ToolchainAgreement(set.Entries("typescript")); len(a.SittingOut) != 1 || a.SittingOut[0].Node != "ace" { // ADR 0584: the majority is agreed, ace's report counts as the one that sits out
|
|
t.Fatalf("before the placement is settled every report counts; got %+v", a)
|
|
}
|
|
k, v := placing(t, "ace", GoBuildModule) // another build module on ace: no matter
|
|
gate.Placement(k, v, false, old)
|
|
if !gate.Settled() {
|
|
t.Fatal("settling left ace's report out, so what was passed on changed")
|
|
}
|
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
|
t.Fatalf("the placed nodes agree on v26.12.1; got %q, %q", v, none)
|
|
}
|
|
if len(*said) != 1 || !strings.Contains((*said)[0], "does not place mesh-build-node on ace") {
|
|
t.Fatalf("ace's report is said left out once; said %q", *said)
|
|
}
|
|
if gate.Toolchain(NodeBuildModule, key, value, false) {
|
|
t.Fatal("a report of a node not placed changed what was passed on")
|
|
}
|
|
// Placed on ace with a report written since: it counts, and the builders really disagree.
|
|
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
|
|
gate.Placement(k, v, false, old.Add(-time.Minute))
|
|
if a := delivery.ToolchainAgreement(set.Entries("typescript")); a.Toolchain.Version != "v26.12.1" || len(a.SittingOut) != 1 || a.SittingOut[0].Version != "v26.12.0" {
|
|
t.Fatalf("builders that run different versions: the majority is agreed and ace sits out; got %+v", a)
|
|
}
|
|
// Taken off ace again (its key deleted): its report goes again.
|
|
if !gate.Placement(k, nil, true, placed) {
|
|
t.Fatal("taking ace's placement away passed nothing on")
|
|
}
|
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
|
t.Fatalf("got %q, %q", v, none)
|
|
}
|
|
// A report deleted from the bucket goes whether its node is placed or not.
|
|
key, _ = gateReport(t, "g14", "v26.12.1", placed)
|
|
if !gate.Toolchain(NodeBuildModule, key, nil, true) || len(set.Entries("typescript")) != 2 {
|
|
t.Fatalf("deleting g14's report: %d entries left, want novox and shanks", len(set.Entries("typescript")))
|
|
}
|
|
})
|
|
|
|
for _, c := range []struct {
|
|
name string
|
|
at time.Time // when ace wrote its report
|
|
counts bool
|
|
}{
|
|
{"a report left before the placement", old, false},
|
|
{"a report written since the placement", placed.Add(30 * time.Second), true},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
gate, set, said := start(t, true)
|
|
k, v := placing(t, "ace") // ace's key, first heard placing nothing
|
|
gate.Placement(k, v, false, old.Add(-time.Hour))
|
|
gate.Settled()
|
|
k, v = placing(t, "ace", NodeBuildModule)
|
|
if gate.Placement(k, v, false, placed) {
|
|
t.Fatal("a placement with no report passed something on")
|
|
}
|
|
key, value := gateReport(t, "ace", "v26.12.0", c.at)
|
|
if got := gate.Toolchain(NodeBuildModule, key, value, false); got != c.counts {
|
|
t.Fatalf("ace's report passed on: %v, want %v", got, c.counts)
|
|
}
|
|
// ace's placement written again with another module beside: its standing does not move, nothing is said again.
|
|
k, v = placing(t, "ace", GoBuildModule, NodeBuildModule)
|
|
if gate.Placement(k, v, false, placed.Add(time.Minute)) {
|
|
t.Fatal("the same placement of mesh-build-node changed what was passed on")
|
|
}
|
|
if c.counts {
|
|
return
|
|
}
|
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
|
t.Fatalf("ace's left report is no builder's; got %q, %q", v, none)
|
|
}
|
|
if len(*said) != 1 || !strings.Contains((*said)[0], "before mesh-build-node was placed on ace at 2026-10-25T19:56:45Z") {
|
|
t.Fatalf("said %q, want once why ace's report is left out", *said)
|
|
}
|
|
key, value = gateReport(t, "ace", "v26.12.1", placed.Add(time.Minute))
|
|
if !gate.Toolchain(NodeBuildModule, key, value, false) || len(set.Entries("typescript")) != 4 {
|
|
t.Fatal("ace's new report was not passed on")
|
|
}
|
|
})
|
|
}
|
|
|
|
t.Run("no placement held: every report counts", func(t *testing.T) {
|
|
gate, set, _ := start(t, false)
|
|
key, value := gateReport(t, "ace", "v26.12.0", old)
|
|
gate.Toolchain(NodeBuildModule, key, value, false)
|
|
gate.Settled()
|
|
if n := len(set.Entries("typescript")); n != 4 {
|
|
t.Fatalf("before the first placement file every report counts; got %d", n)
|
|
}
|
|
k, v := placing(t, "g14", NodeBuildModule)
|
|
if !gate.Placement(k, v, false, old.Add(-time.Hour)) || len(set.Entries("typescript")) != 1 {
|
|
t.Fatalf("the first placement leaves every node it does not place out; got %d", len(set.Entries("typescript")))
|
|
}
|
|
if !gate.Placement(k, nil, true, placed) || len(set.Entries("typescript")) != 4 {
|
|
t.Fatal("the last placement gone: every report counts again")
|
|
}
|
|
})
|
|
|
|
t.Run("the history: a module in the first revision heard of a key stands since before it", func(t *testing.T) {
|
|
p := delivery.NewPlaced()
|
|
// ace in the form before, then converted: its module stands since before, the legacy key read no more.
|
|
k, v := placingLegacy(t, "ace", NodeBuildModule)
|
|
p.Put(k, v, false, old)
|
|
k, v = placing(t, "ace", NodeBuildModule)
|
|
if _, changed, err := p.Put(k, v, false, placed); len(changed) != 0 || err != nil {
|
|
t.Errorf("the conversion of ace's key changed %v, %v", changed, err)
|
|
}
|
|
if _, changed, _ := p.Put("ace."+NodeBuildModule, []byte("{}"), false, placed); len(changed) != 0 {
|
|
t.Errorf("a module key says which modules stand: %v", changed)
|
|
}
|
|
if k, v := placingLegacy(t, "ace"); true {
|
|
if _, changed, _ := p.Put(k, v, false, placed); len(changed) != 0 {
|
|
t.Errorf("the legacy key is read after the node key: %v", changed)
|
|
}
|
|
}
|
|
k2, v2 := placing(t, "g14")
|
|
p.Put(k2, v2, false, old)
|
|
k2, v2 = placing(t, "g14", NodeBuildModule)
|
|
if _, changed, _ := p.Put(k2, v2, false, placed); strings.Join(changed, ",") != NodeBuildModule {
|
|
t.Errorf("a module placed says changed %v", changed)
|
|
}
|
|
if _, _, err := p.Put("placement.novox", []byte("not json"), false, old); err == nil {
|
|
t.Error("a value that does not read was not said")
|
|
}
|
|
if s, ok := p.Since("ace", NodeBuildModule); !ok || !s.IsZero() {
|
|
t.Errorf("ace's module stands since %v, %v; want the zero time", s, ok)
|
|
}
|
|
if s, ok := p.Since("g14", NodeBuildModule); !ok || !s.Equal(placed) {
|
|
t.Errorf("g14's module stands since %v, %v; want the revision that placed it", s, ok)
|
|
}
|
|
if got := p.PlacedNodes(NodeBuildModule); strings.Join(got, ",") != "ace,g14" || len(p.ByNode()) != 2 || !p.Seeded() {
|
|
t.Errorf("PlacedNodes = %v, ByNode = %v", got, p.ByNode())
|
|
}
|
|
})
|
|
|
|
t.Run("a placement the bus refuses is said, and the part goes on", func(t *testing.T) {
|
|
var logs bytes.Buffer
|
|
logger := log.New(&logs, "", 0)
|
|
refused := errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_placement")
|
|
gate, _, _ := start(t, false)
|
|
for _, bus := range []refusingBus{{err: refused}, {refuseOpen: true, err: refused}} {
|
|
if _, err := FollowPlacement(context.Background(), bus, gate, func(context.Context) {}, logger.Printf); err != nil {
|
|
t.Errorf("a refused placement stopped the start: %v", err)
|
|
}
|
|
}
|
|
if !strings.Contains(logs.String(), "cannot be watched") || !strings.Contains(logs.String(), "cannot be read") {
|
|
t.Errorf("the refusals were not said: %q", logs.String())
|
|
}
|
|
})
|
|
}
|
|
|
|
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
|
type refusingBus struct {
|
|
refuseOpen bool
|
|
err error
|
|
}
|
|
|
|
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
|
if r.refuseOpen {
|
|
return nil, r.err
|
|
}
|
|
return refusedWatch{err: r.err}, nil
|
|
}
|
|
|
|
type refusedWatch struct {
|
|
jetstream.KeyValue
|
|
err error
|
|
}
|
|
|
|
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
|
return nil, r.err
|
|
}
|