The builder leaves an event it cannot take for the bus to give again, and the bus drops it after the fifth delivery its durable allows without a word: the 0.1.6 published was lost so. On that last delivery the builder now says it gives the message up, naming its subject, what it carries (a registry outcome's name and version, a version-released's module and version) and the last refusal. Builds: design 47 G A-420
622 lines
32 KiB
Go
622 lines
32 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log"
|
|
"maps"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
|
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient/heard"
|
|
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
|
"git.novox.be/novox/mesh-sdk/go/process"
|
|
"git.novox.be/novox/mesh-sdk/go/views"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
"github.com/nats-io/nats.go/jetstream"
|
|
|
|
"git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
|
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
|
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
|
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
|
"git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds/forms"
|
|
sweep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshartifactsweep"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds"
|
|
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
|
|
|
"mesh-delivery-builder/internal/build"
|
|
"mesh-delivery-builder/internal/versions"
|
|
)
|
|
|
|
// 1945 (a part exits at start for a bucket it works without): a refused bucket it cannot work without is an error naming the bucket and
|
|
// its grant. The gate over the seat's assignments, and its refusal, are mesh-sdk's (buildrun.PlacedNodes,
|
|
// issue 774).
|
|
func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) {
|
|
t.Run("a floor's wake is heard on the events consumer (design 51 The floor)", aFloorsWakeIsHeard)
|
|
t.Run("a failed trunk build is a finding until a later done one (DLV-B7)", aFailedTrunkBuildIsAFinding)
|
|
refused := refusingBus{err: errors.New("nats: permissions violation for publish to $JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules")}
|
|
if _, err := process.EssentialBucket(context.Background(), refusingBus{refuseOpen: true, err: refused.err}, "builder", "mesh-delivery-indexer_modules", "work"); err == nil ||
|
|
!strings.Contains(err.Error(), "grant of mesh-delivery-indexer_modules") {
|
|
t.Errorf("a refused bucket the part needs: %v, want an error naming its grant", err)
|
|
}
|
|
t.Run("ADR 0624: the counted nodes' engine reads", theCountedNodesEngineReads)
|
|
}
|
|
|
|
// theCountedNodesEngineReads: the builder asks the node-engine seat's verb of every node the assignments names; a node away
|
|
// and never asked has no statement and reads as the baseline's first forms, with no node record read in its place; an
|
|
// away node keeps its last statement; a node the assignments no longer names leaves the count, its key removed; and
|
|
// builder_engine_reads drops one node's key.
|
|
func theCountedNodesEngineReads(t *testing.T) {
|
|
gate := buildrun.NewPlacedNodes(t.Logf)
|
|
place := func(nodes ...string) {
|
|
for _, node := range []string{"n1", "n2"} {
|
|
pl := meshmoduleassigner.Assignments{Node: node}
|
|
if slices.Contains(nodes, node) {
|
|
pl.Modules = []meshmoduleassigner.PlacedModule{{Module: "web"}}
|
|
}
|
|
values, err := meshmoduleassignerclient.AssignmentsValues(pl)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for key, raw := range values {
|
|
gate.Assignments(key, raw, false, time.Now())
|
|
}
|
|
if !slices.Contains(nodes, node) {
|
|
key, _ := meshmoduleassignerclient.ModuleKey(node, "web")
|
|
gate.Assignments(key, nil, true, time.Now())
|
|
}
|
|
}
|
|
}
|
|
away := map[string]bool{"n2": true}
|
|
asked := map[string]int{}
|
|
r := newEngineReads(func(node, key string, _ any) (json.RawMessage, error) {
|
|
asked[node]++
|
|
if away[node] {
|
|
return nil, nats.ErrNoResponders
|
|
}
|
|
return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)})
|
|
}, gate, t.Logf)
|
|
r.keeper.Sync = true
|
|
kept := map[string][]byte{}
|
|
r.keeper.Put = func(node string, v []byte) error { kept[node] = v; return nil }
|
|
r.keeper.Delete = func(node string) error { delete(kept, node); return nil }
|
|
place("n1", "n2")
|
|
r.placed()
|
|
present, counted := r.forms()
|
|
if !slices.Equal(present["n1"], []string{"form:new"}) || len(present) != 1 || len(counted) != 2 || counted["n2"] != nil {
|
|
t.Errorf("away and never asked: present %v, counted %v", present, counted)
|
|
}
|
|
if s := r.keeper.Statements()["n2"]; s.From != enginereads.FromNone || s.Answered || asked["n2"] != 1 {
|
|
t.Errorf("n2, away and never asked, is %+v (asked %d)", s, asked["n2"])
|
|
}
|
|
// n2 back, stating {word:module}, then away again: it keeps that statement.
|
|
away["n2"] = false
|
|
r.keeper.Heard("n2", mustStatement(t, enginereads.Statement{Node: "n2", From: enginereads.FromVerb, Answered: true,
|
|
AskAgain: &enginereads.Again{By: "n1.ada", Why: "back"}}), false)
|
|
away["n2"] = true
|
|
r.keeper.AskNode("n2", "its event")
|
|
if _, counted = r.forms(); !slices.Equal(counted["n2"], []string{"form:new"}) || r.keeper.Statements()["n2"].Answered {
|
|
t.Errorf("away again, n2 reads %v (%+v)", counted["n2"], r.keeper.Statements()["n2"])
|
|
}
|
|
// n2 retired: it leaves the assignments, and its key goes.
|
|
place("n1")
|
|
r.placed()
|
|
if _, counted = r.forms(); len(counted) != 1 || kept["n2"] != nil {
|
|
t.Errorf("retired n2: counted %v, its key %s", counted, kept["n2"])
|
|
}
|
|
// builder_engine_reads drops one key.
|
|
st := &memReads{v: map[string][]byte{"n1": kept["n1"]}}
|
|
tool := enginereads.Tool("builder_engine_reads", "the builder", st, true, time.Now)
|
|
if _, err := tool.Run(map[string]any{"node": "n1", "drop": "true", "why": "a test"}); err != nil || st.v["n1"] != nil {
|
|
t.Errorf("drop: %v, the key %s", err, st.v["n1"])
|
|
}
|
|
t.Run("ADR 0668 §3: a manifest-reads call unanswered past five minutes, its machine heard, is call-unanswered", func(t *testing.T) {
|
|
// n2 answers nothing: a bound is kept at five minutes and the watchdogs seat's presence is asked only when it ends
|
|
// (ADR 0596 §2), once; away, no finding; heard, the finding call-unanswered in the builder's findings at the
|
|
// builder's clock; answered, it is cleared.
|
|
place("n1", "n2")
|
|
now := time.Unix(1000, 0)
|
|
clock := func() time.Time { return now }
|
|
down := true
|
|
r2 := newEngineReads(func(node, _ string, _ any) (json.RawMessage, error) {
|
|
if down {
|
|
return nil, nats.ErrNoResponders
|
|
}
|
|
return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: now.Format(time.RFC3339Nano)})
|
|
}, gate, t.Logf)
|
|
r2.keeper.Sync, r2.keeper.Now = true, clock
|
|
st := &memFindings{vals: map[string][]byte{}}
|
|
armed := map[string]time.Time{}
|
|
online, asked := false, []any{}
|
|
heard := heardBy(func(key string, body any) (json.RawMessage, error) {
|
|
asked = append(asked, key, body)
|
|
return json.Marshal(meshwatchdogs.PresenceAnswer{Machine: "n2", Online: online})
|
|
})
|
|
calls := r2.callsOf(&findings{module: builds.BuilderModule, w: st, logf: t.Logf, now: clock}, func() jetstream.KeyValue { return nil }, heard, t.Logf)
|
|
calls.Arm = func(_ context.Context, node string, ends time.Time) error { armed[node] = ends; return nil }
|
|
key := meshconditionsclient.Key(meshconditionsclient.ScopeCall, "manifest-reads.n2", meshconditionsclient.KindCallUnanswered)
|
|
r2.placed() // n2 asked, unanswered at 1000
|
|
if len(st.vals) != 0 || len(asked) != 0 || !armed["n2"].Equal(time.Unix(1000, 0).Add(meshconditionsclient.CallUnansweredAfter)) {
|
|
t.Fatalf("before five minutes: findings %v, presence asked %v, bounds %v", st.vals, asked, armed)
|
|
}
|
|
now = now.Add(meshconditionsclient.CallUnansweredAfter)
|
|
r2.expiredCall("n2")
|
|
if len(st.vals) != 0 || len(asked) != 2 || asked[0] != presenceInvoke || asked[1] != (meshwatchdogs.PresenceInput{Machine: "n2"}) {
|
|
t.Fatalf("its machine away: findings %v, presence asked %v", st.vals, asked)
|
|
}
|
|
online = true
|
|
r2.expiredCall("n2")
|
|
f, err := decodeFinding(st.vals[key])
|
|
if err != nil || f.Observer != builds.BuilderModule || f.Kind != meshconditionsclient.KindCallUnanswered || len(asked) != 4 {
|
|
t.Fatalf("heard five minutes on: finding %+v, %v (keys %v), presence asked %v", f, err, slices.Collect(maps.Keys(st.vals)), asked)
|
|
}
|
|
r2.stepCalls()
|
|
if len(st.vals) != 1 || len(asked) != 4 {
|
|
t.Errorf("a step after it: findings %v, presence asked %v", st.vals, asked)
|
|
}
|
|
down = false
|
|
r2.keeper.AskNode("n2", "answered now, its manifest-reads-changed heard")
|
|
if len(st.vals) != 0 {
|
|
t.Errorf("answered leaves %v", st.vals)
|
|
}
|
|
})
|
|
}
|
|
|
|
func mustStatement(t *testing.T, s enginereads.Statement) []byte {
|
|
t.Helper()
|
|
raw, err := enginereads.Encode(s)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return raw
|
|
}
|
|
|
|
// memReads is the builder's state of engine reads in memory, as the tool reads it.
|
|
type memReads struct{ v map[string][]byte }
|
|
|
|
func (m *memReads) Keys() ([]string, error) { return slices.Collect(maps.Keys(m.v)), nil }
|
|
func (m *memReads) Get(key string) ([]byte, uint64, bool, error) {
|
|
v, ok := m.v[key]
|
|
return v, 1, ok, nil
|
|
}
|
|
func (m *memReads) Put(key string, value []byte) (uint64, error) { m.v[key] = value; return 2, nil }
|
|
func (m *memReads) Delete(key string) error { delete(m.v, key); return nil }
|
|
func (m *memReads) Await(string, uint64, func([]byte) bool, time.Duration) ([]byte, error) {
|
|
return nil, errors.New("no part answers here")
|
|
}
|
|
|
|
// refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err.
|
|
type refusingBus struct {
|
|
refuseOpen bool
|
|
err error
|
|
}
|
|
|
|
func (r refusingBus) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
|
|
if r.refuseOpen {
|
|
return nil, r.err
|
|
}
|
|
return refusedWatch{err: r.err}, nil
|
|
}
|
|
|
|
type refusedWatch struct {
|
|
jetstream.KeyValue
|
|
err error
|
|
}
|
|
|
|
func (r refusedWatch) WatchAll(context.Context, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
|
return nil, r.err
|
|
}
|
|
func (r refusedWatch) Watch(context.Context, string, ...jetstream.WatchOpt) (jetstream.KeyWatcher, error) {
|
|
return nil, r.err
|
|
}
|
|
|
|
// wakeMsg is a message of the builder's events consumer: its subject and body, acknowledged.
|
|
type wakeMsg struct {
|
|
jetstream.Msg
|
|
subject string
|
|
data []byte
|
|
acked bool
|
|
naked bool
|
|
delayed time.Duration
|
|
// delivered is the delivery count Metadata says; 0 says no metadata.
|
|
delivered uint64
|
|
}
|
|
|
|
func (m *wakeMsg) Subject() string { return m.subject }
|
|
func (m *wakeMsg) Data() []byte { return m.data }
|
|
func (m *wakeMsg) Ack() error { m.acked = true; return nil }
|
|
func (m *wakeMsg) Nak() error { m.naked = true; return nil }
|
|
func (m *wakeMsg) NakWithDelay(d time.Duration) error {
|
|
m.delayed = d
|
|
return nil
|
|
}
|
|
func (m *wakeMsg) Metadata() (*jetstream.MsgMetadata, error) {
|
|
if m.delivered > 0 {
|
|
return &jetstream.MsgMetadata{NumDelivered: m.delivered}, nil
|
|
}
|
|
return nil, errors.New("no metadata")
|
|
}
|
|
func (m *wakeMsg) Headers() nats.Header {
|
|
return nil
|
|
}
|
|
|
|
func aFloorsWakeIsHeard(t *testing.T) {
|
|
var logs bytes.Buffer
|
|
holders := map[string]string{"git": "mesh-delivery-forge-bridge", "mesh-catalog": "mesh-delivery-indexer"}
|
|
waits := &versions.FloorWaits{Holder: func(seat string) (string, bool) { h, ok := holders[seat]; return h, ok }}
|
|
waits.Wait("m/lib", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceRequirements}}, nil, time.Unix(0, 0))
|
|
waits.Wait("m/img", versions.SeatImages, "", versions.Floor{Silent: []string{versions.SourceTags}}, nil, time.Unix(0, 0))
|
|
f := &floorWake{b: &build.Builder{}, waits: waits, logf: log.New(&logs, "", 0).Printf}
|
|
accepted := &wakeMsg{subject: manifestAcceptedSubject}
|
|
if !f.heard(context.Background(), accepted) || !accepted.acked || !strings.Contains(logs.String(), "the floor of m/lib is asked again: manifest-accepted") {
|
|
t.Fatalf("manifest-accepted: acked %v, log %q", accepted.acked, logs.String())
|
|
}
|
|
logs.Reset()
|
|
cleared := &wakeMsg{subject: conditionClearedSubject, data: []byte(`{"subject":{"scope":"probe","id":"ace/mesh-delivery-forge-bridge"}}`)}
|
|
if !f.heard(context.Background(), cleared) || !strings.Contains(logs.String(), "the floor of m/img is asked again: condition-cleared") {
|
|
t.Fatalf("condition-cleared: log %q", logs.String())
|
|
}
|
|
if f.heard(context.Background(), &wakeMsg{subject: sweep.SubjectArtifactsGone}) {
|
|
t.Fatal("artifacts-gone was taken as a floor's wake")
|
|
}
|
|
// DLV-B4: a registry seat's outcome on the same consumer moves a version; a published wakes a name whose seat was
|
|
// silent; one the state cannot take is left for the bus to give again.
|
|
logs.Reset()
|
|
waits.Wait("m/pkg", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceSeat}}, nil, time.Unix(0, 0))
|
|
h := &seatHearer{n: &versions.Numberer{Versions: versions.Versions{Store: noVersions{}}, Waits: waits}, b: &build.Builder{}, logf: log.New(&logs, "", 0).Printf}
|
|
byDigest := &wakeMsg{subject: "mesh.seat.go-package-registry.event.published", data: []byte(`{"name":"m/other","digest":"sha256:aa","kind":"package","holder":"gitea"}`)}
|
|
if !h.heard(context.Background(), byDigest) || !byDigest.acked || !strings.Contains(logs.String(), "the floor of m/pkg is asked again: go-package-registry said published") {
|
|
t.Fatalf("published: acked %v, log %q", byDigest.acked, logs.String())
|
|
}
|
|
// Design 47 G §369, A-415: a published of a version the reservation numbered, which the state versions does not hold, is
|
|
// taken: its artifact is marked and package-published said, never left for the bus to give again.
|
|
var saidPub []meshbuilds.PackagePublished
|
|
rv := newReservations(&memReserve{vals: map[string][]byte{}, revs: map[string]uint64{}}, func(string, any) (json.RawMessage, error) {
|
|
return json.Marshal(meshcatalog.VersionsAnswer{Module: "m"})
|
|
}, func() jetstream.KeyValue { return &boundsKV{keys: map[string]bool{}} },
|
|
&findings{module: builds.BuilderModule, w: &memFindings{vals: map[string][]byte{}}, logf: t.Logf, now: time.Now}, nil, t.Logf)
|
|
rv.say = func(_ context.Context, p meshbuilds.PackagePublished) error { saidPub = append(saidPub, p); return nil }
|
|
rv.filled.Store(true)
|
|
res, err := rv.Reserve(context.Background(), "m", "0.1", []string{"m:program"}, versions.Source{})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := rv.Built(context.Background(), "m", res.Version, map[string]string{"m:program": "sha256:bb"}, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
h.res = rv
|
|
published := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", data: []byte(`{"name":"m/program","version":"` + res.Version + `","digest":"sha256:bb","kind":"bundle"}`)}
|
|
if !h.heard(context.Background(), published) || !published.acked || published.naked || len(saidPub) != 1 || saidPub[0].Artifact != "m:program" {
|
|
t.Fatalf("a published of a reserved version: acked %v, naked %v, said %+v, log %q", published.acked, published.naked, saidPub, logs.String())
|
|
}
|
|
if h.heard(context.Background(), &wakeMsg{subject: manifestAcceptedSubject}) {
|
|
t.Fatal("manifest-accepted was taken as a seat's outcome")
|
|
}
|
|
// Design 47 G A-420: a message left for the bus is said given up on the last delivery its consumer allows, never before.
|
|
for _, n := range []uint64{4, EventDeliveries} {
|
|
logs.Reset()
|
|
givenUp(&wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", delivered: n}, "oci-artifact-registry's published of m/program@0.1.6", errors.New("down"), log.New(&logs, "", 0).Printf)
|
|
said := strings.Contains(logs.String(), "gives up mesh.seat.oci-artifact-registry.event.published: oci-artifact-registry's published of m/program@0.1.6, delivered 5 times, the most its consumer allows; the last refusal: down")
|
|
if said != (n == EventDeliveries) {
|
|
t.Errorf("delivery %d: log %q", n, logs.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
// noVersions is a state versions that holds nothing.
|
|
type noVersions struct{}
|
|
|
|
func (noVersions) Get(context.Context, string) ([]byte, uint64, bool, error) {
|
|
return nil, 0, false, nil
|
|
}
|
|
func (noVersions) Create(context.Context, string, []byte) (uint64, error) {
|
|
return 0, errors.New("read only")
|
|
}
|
|
func (noVersions) Update(context.Context, string, []byte, uint64) (uint64, error) {
|
|
return 0, errors.New("read only")
|
|
}
|
|
func (noVersions) KeysOf(context.Context, string) ([]string, error) { return nil, nil }
|
|
|
|
// failingVersions is a state versions that cannot be read.
|
|
type failingVersions struct{ noVersions }
|
|
|
|
func (failingVersions) KeysOf(context.Context, string) ([]string, error) {
|
|
return nil, errors.New("down")
|
|
}
|
|
|
|
func aFailedTrunkBuildIsAFinding(t *testing.T) {
|
|
st := &memFindings{vals: map[string][]byte{}}
|
|
f := &findings{module: builds.BuilderModule, w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
|
ended := func(outcome, ref string) builds.BuildLogEntry {
|
|
return builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "m", FP: "f1", Commit: "c", Ref: ref, Outcome: outcome}
|
|
}
|
|
key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m", BuildFailed)
|
|
f.buildEnded(context.Background(), ended(builds.BuildFailed, "refs/pull/3/head"))
|
|
if len(st.vals) != 0 {
|
|
t.Fatalf("a candidate's failed build is a finding: %v", st.vals)
|
|
}
|
|
f.buildEnded(context.Background(), ended(builds.BuildFailed, ""))
|
|
got, err := decodeFinding(st.vals[key])
|
|
if err != nil || got.Observer != builds.BuilderModule || !strings.Contains(got.Summary, "trunk build of m at f1") {
|
|
t.Fatalf("the finding = %+v, %v", got, err)
|
|
}
|
|
f.buildEnded(context.Background(), ended(builds.BuildDone, ""))
|
|
if _, ok := st.vals[key]; ok {
|
|
t.Error("a later done trunk build leaves build-failed")
|
|
}
|
|
// The floor finding: raised while a name waits on its floor, cleared once it is known.
|
|
fw := &versions.FloorWaits{Said: f.floorSaid(context.Background())}
|
|
fw.Wait("m/lib", versions.SeatGoPackages, "", versions.Floor{Silent: []string{versions.SourceTags}}, nil, time.Unix(0, 0))
|
|
floorKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m/lib", versions.FloorSilent)
|
|
if got, err := decodeFinding(st.vals[floorKey]); err != nil || !strings.Contains(got.Summary, "git.tags did not answer") {
|
|
t.Fatalf("the floor finding = %+v, %v", got, err)
|
|
}
|
|
fw.Answered("m/lib")
|
|
if _, ok := st.vals[floorKey]; ok {
|
|
t.Error("a known floor leaves floor-silent")
|
|
}
|
|
// Design 51 *Health and conditions* (review round 1): a holder's refused is publish-refused with its reason, a version
|
|
// no holder answered within its bound publish-unanswered, a seat's outcome that does not read seat-outcome-unread;
|
|
// a later published of the name clears the first two, the seat's next outcome that reads the third.
|
|
vs := versions.Versions{Store: memVersions{}}
|
|
for _, v := range []string{"0.1.3", "0.1.4"} {
|
|
raw, err := versions.Encode(versions.Entry{Name: "m/program", Kind: versions.KindBundle, Series: "0.1", Version: v, Seat: "oci-artifact-registry", State: versions.Numbered})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
vs.Store.(memVersions)[versions.Key("m/program", v)] = raw
|
|
}
|
|
pubs := &publishes{bounds: func() jetstream.KeyValue { return nil }, versions: vs, findings: f, bound: versions.PublishBound, logf: t.Logf}
|
|
h := &seatHearer{n: &versions.Numberer{Versions: vs}, b: &build.Builder{}, logf: t.Logf, pub: pubs}
|
|
refusedKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m/program", versions.PublishRefused)
|
|
refused := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.refused", data: []byte(`{"name":"m/program","version":"0.1.3","digest":"sha256:bb","reason":"bytes differ"}`)}
|
|
if !h.heard(context.Background(), refused) || !refused.acked {
|
|
t.Fatalf("a refusal: acked %v", refused.acked)
|
|
}
|
|
if got, err := decodeFinding(st.vals[refusedKey]); err != nil || !strings.Contains(got.Summary, "refused to publish m/program@0.1.3") || !strings.Contains(got.Summary, "bytes differ") {
|
|
t.Fatalf("a holder's refused: the finding = %+v, %v", got, err)
|
|
}
|
|
pubs.ended(context.Background(), purged{key: versions.PublishBoundKey("m/program", "0.1.4")})
|
|
unansweredKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m/program", versions.PublishUnanswered)
|
|
if got, err := decodeFinding(st.vals[unansweredKey]); err != nil || !strings.Contains(got.Summary, "neither published nor refused of m/program@0.1.4") {
|
|
t.Fatalf("a bound ended unanswered: the finding = %+v, %v", got, err)
|
|
}
|
|
// Review round 2: a bound that cannot be written is publish-unanswered at once; a watch of bounds that cannot start
|
|
// is family-not-heard naming the state.
|
|
delete(st.vals, unansweredKey)
|
|
pubs.arm(context.Background(), versions.Entry{Name: "m/program", Version: "0.1.4", Seat: "oci-artifact-registry"})
|
|
if got, err := decodeFinding(st.vals[unansweredKey]); err != nil || !strings.Contains(got.Summary, "its bound could not be written") {
|
|
t.Fatalf("a bound not written: the finding = %+v, %v", got, err)
|
|
}
|
|
watchBounds(context.Background(), refusedWatch{err: errors.New("permissions violation")}, func(jetstream.KeyValueEntry) {}, f.notHeard(context.Background()), nil, t.Logf)
|
|
if got, err := decodeFinding(st.vals[heard.Key(builds.BuilderModule, versions.BoundsState)]); err != nil || !strings.Contains(got.Summary, "permissions violation") {
|
|
t.Fatalf("bounds not watched: the finding = %+v, %v", got, err)
|
|
}
|
|
delete(st.vals, heard.Key(builds.BuilderModule, versions.BoundsState))
|
|
malformed := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", data: []byte(`{"version":"0.1.4"}`)}
|
|
unreadKey := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "oci-artifact-registry", versions.SeatOutcomeUnread)
|
|
if !h.heard(context.Background(), malformed) || !malformed.acked || st.vals[unreadKey] == nil {
|
|
t.Fatalf("an outcome that does not read: acked %v, findings %v", malformed.acked, st.vals)
|
|
}
|
|
published := &wakeMsg{subject: "mesh.seat.oci-artifact-registry.event.published", data: []byte(`{"name":"m/program","version":"0.1.4","digest":"sha256:cc"}`)}
|
|
if !h.heard(context.Background(), published) || st.vals[refusedKey] != nil || st.vals[unansweredKey] != nil || st.vals[unreadKey] != nil {
|
|
t.Fatalf("a later published leaves the findings: %v", st.vals)
|
|
}
|
|
// Issue 1966 (ADR 0668 §3, §5; 1748 note 27): a key a look left building because its end could not be written arms its
|
|
// bound; the bound's end while the key is still building and no look holds it is call-unanswered about the key; the key's
|
|
// next written end deletes the bound and clears it; a key a look holds, or one no longer building, raises nothing; a
|
|
// bound that cannot be written raises at once.
|
|
buildKey := "m.1111111111111111111111111111111111111111111111111111111111111111"
|
|
building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "novox.builder", Step: "claimed"})
|
|
done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, By: "novox.builder", Step: "built"})
|
|
keys := memBuilds{memVersions{buildKey: building}}
|
|
looking := false
|
|
unputMarks := &memBounds{keys: map[string]bool{}}
|
|
u := &unputs{bounds: func() jetstream.KeyValue { return unputMarks }, store: keys, looking: func(string) bool { return looking },
|
|
findings: f, bound: meshconditionsclient.CallUnansweredAfter, now: func() time.Time { return time.Unix(600, 0) }, logf: t.Logf}
|
|
callKey := meshconditionsclient.Key(meshconditionsclient.ScopeCall, buildKey, meshconditionsclient.KindCallUnanswered)
|
|
u.left(context.Background(), buildKey, "nats: timeout")
|
|
if !unputMarks.keys[versions.UnputBoundKey(buildKey)] || st.vals[callKey] != nil {
|
|
t.Fatalf("a key left building: bound %v, finding %v; want the bound armed, no finding yet", unputMarks.keys, st.vals[callKey] != nil)
|
|
}
|
|
for name, c := range map[string]struct {
|
|
value []byte
|
|
look bool
|
|
}{"a look holds it": {building, true}, "no longer building": {done, false}} {
|
|
keys.memVersions[buildKey], looking = c.value, c.look
|
|
u.expired(context.Background(), purged{key: versions.UnputBoundKey(buildKey)})
|
|
if st.vals[callKey] != nil {
|
|
t.Errorf("%s: the bound's end raised call-unanswered", name)
|
|
}
|
|
}
|
|
keys.memVersions[buildKey], looking = building, false
|
|
u.expired(context.Background(), purged{key: versions.UnputBoundKey(buildKey)})
|
|
if got, err := decodeFinding(st.vals[callKey]); err != nil || got.Kind != meshconditionsclient.KindCallUnanswered || !strings.Contains(got.Summary, buildKey) || !strings.Contains(got.Needs, build.BuildAgainTool) {
|
|
t.Fatalf("the bound's end of a key left building: the finding = %+v, %v", got, err)
|
|
}
|
|
u.ended(context.Background(), buildKey)
|
|
if unputMarks.keys[versions.UnputBoundKey(buildKey)] || st.vals[callKey] != nil {
|
|
t.Errorf("the key's end written: bound %v, finding left %v", unputMarks.keys, st.vals[callKey] != nil)
|
|
}
|
|
u.bounds = func() jetstream.KeyValue { return nil }
|
|
u.left(context.Background(), buildKey, "nats: timeout")
|
|
if got, err := decodeFinding(st.vals[callKey]); err != nil || !strings.Contains(got.Explanation, "nor its bound") {
|
|
t.Fatalf("a bound not written: the finding = %+v, %v", got, err)
|
|
}
|
|
delete(st.vals, callKey)
|
|
// ADR 0615 §5 (review round 1): a consumer not there yet is family-not-heard naming its stream, cleared when it binds.
|
|
notHeard := f.notHeard(context.Background())
|
|
unheardKey := heard.Key(builds.BuilderModule, "EVENTS")
|
|
_ = notHeard.Bind("EVENTS", "a branch's end is not heard", func() error { return errors.New("consumer not found") })
|
|
if got, err := decodeFinding(st.vals[unheardKey]); err != nil || got.Kind != heard.Kind || !strings.Contains(got.Summary, "does not hear EVENTS") {
|
|
t.Fatalf("a consumer not there: the finding = %+v, %v", got, err)
|
|
}
|
|
_ = notHeard.Bind("EVENTS", "a branch's end is not heard", func() error { return nil })
|
|
if _, ok := st.vals[unheardKey]; ok {
|
|
t.Error("a consumer bound leaves family-not-heard")
|
|
}
|
|
// DLV-B9, the old form's window: one counted node reading only the first form holds it; seven days on it is
|
|
// old-form-held about that node; the node stating the newest form closes it, said once, the finding cleared; a
|
|
// retired node dropped by hand leaves the count.
|
|
now := time.Unix(0, 0)
|
|
fs := forms.ArtifactForms{{Name: "registry", First: true, Kinds: []string{"image"}}, {Name: "oci", Kinds: []string{"image"}}}
|
|
var closings []versions.OldFormClosedBody
|
|
var logged []builds.BuildLogEntry
|
|
marks := &memBounds{keys: map[string]bool{}}
|
|
window := &oldForm{module: builds.BuilderModule, forms: fs, findings: f, logf: t.Logf, now: func() time.Time { return now }, bounds: func() jetstream.KeyValue { return marks },
|
|
said: func(b versions.OldFormClosedBody) { closings = append(closings, b) },
|
|
log: func(e builds.BuildLogEntry) { logged = append(logged, e) }}
|
|
reads := &engineReads{module: builds.BuilderModule, keeper: &enginereads.Keeper{}, window: window, counted: []string{"a", "b"}}
|
|
// Issue 1966 note 155, ADR 0668 §4: at the consumer's bind every counted node is called once again.
|
|
var boundCalls []string
|
|
reads.keeper.Sync = true
|
|
reads.keeper.Call = func(node, _ string, _ any) (json.RawMessage, error) {
|
|
boundCalls = append(boundCalls, node)
|
|
return nil, errors.New("not served")
|
|
}
|
|
reads.atBind(now)
|
|
if !slices.Equal(boundCalls, []string{"a", "b"}) {
|
|
t.Fatalf("at the bind called %v, want every counted node once", boundCalls)
|
|
}
|
|
reads.keeper = &enginereads.Keeper{}
|
|
newest := []string{forms.FormNeed("oci")}
|
|
counted := map[string][]string{"a": newest, "b": nil}
|
|
window.step(context.Background(), []string{"a", "b"}, counted)
|
|
heldKey := meshconditionsclient.Key(meshconditionsclient.ScopeMachine, "b", versions.OldFormHeld)
|
|
if !window.w.Open || !slices.Equal(window.w.Holding, []string{"b"}) || st.vals[heldKey] != nil {
|
|
t.Fatalf("one node lacking the newest form: %+v", window.w)
|
|
}
|
|
// Review round 1: the node holding it has a bound in the state bounds ending at its seven days, so old-form-held is
|
|
// raised then, not at the next change.
|
|
if !marks.keys[versions.OldFormBoundKey("b")] || marks.keys[versions.OldFormBoundKey("a")] {
|
|
t.Fatalf("the old form's bounds = %v, want b's alone", marks.keys)
|
|
}
|
|
// Security-upgrade F3 (review round 1): an empty node_haves read, naming no node, leaves the window open.
|
|
window.step(context.Background(), nil, map[string][]string{})
|
|
if !window.w.Open || !slices.Equal(window.w.Holding, []string{"b"}) || len(closings) != 0 {
|
|
t.Fatalf("an empty read closed the window: %+v, closings %+v", window.w, closings)
|
|
}
|
|
now = now.Add(versions.FindingAfter)
|
|
window.step(context.Background(), []string{"a", "b"}, counted)
|
|
if got, err := decodeFinding(st.vals[heldKey]); err != nil || !strings.Contains(got.Summary, "the node-engine of b") {
|
|
t.Fatalf("seven days held: the finding = %+v, %v", got, err)
|
|
}
|
|
counted["b"] = newest
|
|
window.step(context.Background(), []string{"a", "b"}, counted)
|
|
window.step(context.Background(), []string{"a", "b"}, counted)
|
|
if marks.keys[versions.OldFormBoundKey("b")] {
|
|
t.Error("a node no longer holding the window keeps its bound")
|
|
}
|
|
if window.w.Open || len(closings) != 1 || !slices.Equal(closings[0].LastHolding, []string{"b"}) || st.vals[heldKey] != nil {
|
|
t.Errorf("the node back with the newest form: %+v, closings %+v, finding left %v", window.w, closings, st.vals[heldKey] != nil)
|
|
}
|
|
if len(logged) != 1 || logged[0].Event != builds.BuildSaidEvent || logged[0].Why != versions.SaidOldFormClosed ||
|
|
!strings.Contains(logged[0].Line, "last held by b") {
|
|
t.Errorf("the build log = %+v, want the close said once (build.said)", logged)
|
|
} else if _, err := builds.EncodeBuildLogEntry(logged[0]); err != nil {
|
|
t.Errorf("the close is no entry the build log takes: %v", err)
|
|
}
|
|
if _, err := window.drop(context.Background(), "b", "op@shanks", "retired", now); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, c := reads.forms(); len(c) != 1 || c["b"] != nil {
|
|
t.Errorf("a node dropped by hand is still counted: %v", c)
|
|
}
|
|
}
|
|
|
|
// memVersions is a state versions in memory.
|
|
type memVersions map[string][]byte
|
|
|
|
func (m memVersions) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
|
|
v, ok := m[key]
|
|
return v, 1, ok, nil
|
|
}
|
|
func (m memVersions) Create(_ context.Context, key string, v []byte) (uint64, error) {
|
|
m[key] = v
|
|
return 1, nil
|
|
}
|
|
func (m memVersions) Update(_ context.Context, key string, v []byte, _ uint64) (uint64, error) {
|
|
m[key] = v
|
|
return 2, nil
|
|
}
|
|
func (m memVersions) KeysOf(context.Context, string) ([]string, error) {
|
|
return slices.Collect(maps.Keys(m)), nil
|
|
}
|
|
|
|
// memBuilds is the builder's builds in memory.
|
|
type memBuilds struct{ memVersions }
|
|
|
|
func (m memBuilds) Keys(context.Context) ([]string, error) {
|
|
return slices.Collect(maps.Keys(m.memVersions)), nil
|
|
}
|
|
|
|
// memBounds is the state bounds in memory: which keys stand.
|
|
type memBounds struct {
|
|
jetstream.KeyValue
|
|
keys map[string]bool
|
|
}
|
|
|
|
func (m *memBounds) Create(_ context.Context, key string, _ []byte, _ ...jetstream.KVCreateOpt) (uint64, error) {
|
|
if m.keys[key] {
|
|
return 0, jetstream.ErrKeyExists
|
|
}
|
|
m.keys[key] = true
|
|
return 1, nil
|
|
}
|
|
func (m *memBounds) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
|
delete(m.keys, key)
|
|
return nil
|
|
}
|
|
|
|
// purged is the bus's marker of a key whose time to live ended.
|
|
type purged struct {
|
|
jetstream.KeyValueEntry
|
|
key string
|
|
}
|
|
|
|
func (p purged) Key() string { return p.key }
|
|
func (p purged) Operation() jetstream.KeyValueOp { return jetstream.KeyValuePurge }
|
|
|
|
// memFindings is a findings bucket in memory.
|
|
type memFindings struct{ vals map[string][]byte }
|
|
|
|
// Write keeps the finding under its key, as the held writer does in the state.
|
|
func (m *memFindings) Write(_ context.Context, f mcseat.Finding) error {
|
|
v, err := json.Marshal(f)
|
|
m.vals[views.FindingKey(f.Scope, f.ID, f.Kind)] = v
|
|
return err
|
|
}
|
|
|
|
// Clear deletes the finding of scope, id and kind.
|
|
func (m *memFindings) Clear(_ context.Context, scope, id, kind, _ string) error {
|
|
delete(m.vals, views.FindingKey(scope, id, kind))
|
|
return nil
|
|
}
|
|
|
|
// decodeFinding reads a finding as the writer kept it.
|
|
func decodeFinding(v []byte) (mcseat.Finding, error) {
|
|
var f mcseat.Finding
|
|
return f, json.Unmarshal(v, &f)
|
|
}
|
|
|
|
// Design 47 G A-296: the builder's findings are observed by the module its credential names, never a constant of its name.
|
|
func TestTheBuildersFindingsAreObservedByItsCredentialsModule(t *testing.T) {
|
|
st := &memFindings{vals: map[string][]byte{}}
|
|
f := &findings{module: "acme-builder", w: st, logf: t.Logf, now: func() time.Time { return time.Unix(0, 0) }}
|
|
f.buildEnded(context.Background(), builds.BuildLogEntry{Event: builds.BuildEndedEvent, Module: "m", FP: "f1", Commit: "c", Outcome: builds.BuildFailed})
|
|
got, err := decodeFinding(st.vals[meshconditionsclient.Key(meshconditionsclient.ScopeModule, "m", BuildFailed)])
|
|
if err != nil || got.Observer != "acme-builder" {
|
|
t.Fatalf("the finding = %+v, %v; want the observer acme-builder", got, err)
|
|
}
|
|
}
|